gff_loader.inc 75 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458145914601461146214631464146514661467146814691470147114721473147414751476147714781479148014811482148314841485148614871488148914901491149214931494149514961497149814991500150115021503150415051506150715081509151015111512151315141515151615171518151915201521152215231524152515261527152815291530153115321533153415351536153715381539154015411542154315441545154615471548154915501551155215531554155515561557155815591560156115621563156415651566156715681569157015711572157315741575157615771578157915801581158215831584158515861587158815891590159115921593159415951596159715981599160016011602160316041605160616071608160916101611161216131614161516161617161816191620162116221623162416251626162716281629163016311632163316341635163616371638163916401641164216431644164516461647164816491650165116521653165416551656165716581659166016611662166316641665166616671668166916701671167216731674167516761677167816791680168116821683168416851686168716881689169016911692169316941695169616971698169917001701170217031704170517061707170817091710171117121713171417151716171717181719172017211722172317241725172617271728172917301731173217331734173517361737173817391740174117421743174417451746174717481749175017511752175317541755175617571758175917601761176217631764176517661767176817691770177117721773177417751776177717781779178017811782178317841785178617871788178917901791179217931794179517961797179817991800180118021803180418051806180718081809181018111812181318141815181618171818181918201821182218231824182518261827182818291830183118321833183418351836183718381839184018411842184318441845184618471848184918501851185218531854185518561857185818591860186118621863186418651866186718681869187018711872187318741875187618771878187918801881188218831884188518861887188818891890189118921893189418951896189718981899190019011902190319041905190619071908190919101911191219131914191519161917191819191920192119221923192419251926192719281929193019311932193319341935193619371938193919401941194219431944194519461947194819491950195119521953195419551956195719581959196019611962196319641965196619671968196919701971197219731974197519761977197819791980198119821983198419851986198719881989199019911992199319941995
  1. <?php
  2. /**
  3. * @file
  4. * @todo Add file header description
  5. */
  6. /**
  7. * @defgroup gff3_loader GFF3 Feature Loader
  8. * @ingroup tripal_feature
  9. * @{
  10. * Provides gff3 loading functionality. Creates features based on their specification in a GFF3 file.
  11. * @}
  12. */
  13. /**
  14. *
  15. *
  16. * @ingroup gff3_loader
  17. */
  18. function tripal_feature_gff3_load_form() {
  19. $form['gff_file']= array(
  20. '#type' => 'textfield',
  21. '#title' => t('GFF3 File'),
  22. '#description' => t('Please enter the full system path for the GFF file, or a path within the Drupal
  23. installation (e.g. /sites/default/files/xyz.gff). The path must be accessible to the
  24. server on which this Drupal instance is running.'),
  25. '#required' => TRUE,
  26. );
  27. // get the list of organisms
  28. $sql = "SELECT * FROM {organism} ORDER BY genus, species";
  29. $org_rset = chado_query($sql);
  30. $organisms = array();
  31. $organisms[''] = '';
  32. while ($organism = $org_rset->fetchObject()) {
  33. $organisms[$organism->organism_id] = "$organism->genus $organism->species ($organism->common_name)";
  34. }
  35. $form['organism_id'] = array(
  36. '#title' => t('Organism'),
  37. '#type' => t('select'),
  38. '#description' => t("Choose the organism to which these sequences are associated"),
  39. '#required' => TRUE,
  40. '#options' => $organisms,
  41. );
  42. // get the list of analyses
  43. $sql = "SELECT * FROM {analysis} ORDER BY name";
  44. $org_rset = chado_query($sql);
  45. $analyses = array();
  46. $analyses[''] = '';
  47. while ($analysis = $org_rset->fetchObject()) {
  48. $analyses[$analysis->analysis_id] = "$analysis->name ($analysis->program $analysis->programversion, $analysis->sourcename)";
  49. }
  50. $form['analysis_id'] = array(
  51. '#title' => t('Analysis'),
  52. '#type' => t('select'),
  53. '#description' => t("Choose the analysis to which these features are associated.
  54. Why specify an analysis for a data load? All data comes
  55. from some place, even if downloaded from Genbank. By specifying
  56. analysis details for all data imports it allows an end user to reproduce the
  57. data set, but at least indicates the source of the data."),
  58. '#required' => TRUE,
  59. '#options' => $analyses,
  60. );
  61. $form['line_number']= array(
  62. '#type' => 'textfield',
  63. '#title' => t('Start Line Number'),
  64. '#description' => t('Enter the line number in the GFF file where you would like to begin processing. The
  65. first line is line number 1. This option is useful for examining loading problems with large GFF files.'),
  66. '#size' => 10,
  67. );
  68. $form['landmark_type'] = array(
  69. '#title' => t('Landmark Type'),
  70. '#type' => t('textfield'),
  71. '#description' => t("Optional. Use this field to specify a Sequence Ontology type
  72. for the landmark sequences in the GFF fie (e.g. 'chromosome'). If the GFF file
  73. contains a '##sequence-region' line that describes the landmark sequences to
  74. which all others are aligned and a type is provided here then the features
  75. will be created if they do not already exist. If they do exist then this
  76. field is not used."),
  77. );
  78. $form['alt_id_attr'] = array(
  79. '#title' => t('ID Attribute'),
  80. '#type' => t('textfield'),
  81. '#description' => t("Optional. Sometimes lines in the GFF file are missing the
  82. required ID attribute that specifies the unique name of the feature. If so,
  83. you may specify an the name of an existing attribute to use for the name."),
  84. );
  85. $form['import_options'] = array(
  86. '#type' => 'fieldset',
  87. '#title' => t('Import Options'),
  88. '#collapsed' => TRUE
  89. );
  90. $form['import_options']['use_transaction']= array(
  91. '#type' => 'checkbox',
  92. '#title' => t('Use a transaction'),
  93. '#required' => FALSE,
  94. '#description' => t('Use a database transaction when loading the GFF file. If an error occurs
  95. the entire datset loaded prior to the failure will be rolled back and will not be available
  96. in the database. If this option is unchecked and failure occurs all records up to the point
  97. of failure will be present in the database.'),
  98. '#default_value' => 1,
  99. );
  100. $form['import_options']['add_only']= array(
  101. '#type' => 'checkbox',
  102. '#title' => t('Import only new features'),
  103. '#required' => FALSE,
  104. '#description' => t('The job will skip features in the GFF file that already
  105. exist in the database and import only new features.'),
  106. );
  107. $form['import_options']['update']= array(
  108. '#type' => 'checkbox',
  109. '#title' => t('Import all and update'),
  110. '#required' => FALSE,
  111. '#default_value' => 'checked',
  112. '#description' => t('Existing features will be updated and new features will be added. Attributes
  113. for a feature that are not present in the GFF but which are present in the
  114. database will not be altered.'),
  115. );
  116. $form['import_options']['refresh']= array(
  117. '#type' => 'checkbox',
  118. '#title' => t('Import all and replace'),
  119. '#required' => FALSE,
  120. '#description' => t('Existing features will be updated and feature properties not
  121. present in the GFF file will be removed.'),
  122. );
  123. $form['import_options']['remove']= array(
  124. '#type' => 'checkbox',
  125. '#title' => t('Delete features'),
  126. '#required' => FALSE,
  127. '#description' => t('Features present in the GFF file that exist in the database
  128. will be removed rather than imported'),
  129. );
  130. $form['import_options']['create_organism']= array(
  131. '#type' => 'checkbox',
  132. '#title' => t('Create organism'),
  133. '#required' => FALSE,
  134. '#description' => t('The Tripal GFF loader supports the "organism" attribute. This allows features of a
  135. different organism to be aligned to the landmark sequence of another species. The format of the
  136. attribute is "organism=[genus]:[species]", where [genus] is the organism\'s genus and [species] is the
  137. species name. Check this box to automatically add the organism to the database if it does not already exists.
  138. Otherwise lines with an oraganism attribute where the organism is not present in the database will be skipped.'),
  139. );
  140. $form['targets'] = array(
  141. '#type' => 'fieldset',
  142. '#title' => t('Targets'),
  143. '#collapsed' => TRUE
  144. );
  145. $form['targets']['adesc'] = array(
  146. '#type' => 'markup',
  147. '#value' => t("When alignments are represented in the GFF file (e.g. such as
  148. alignments of cDNA sequences to a whole genome, or blast matches), they are
  149. represented using two feature types: 'match' (or cDNA_match, EST_match, etc.)
  150. and 'match_part'. These features may also have a 'Target' attribute to
  151. specify the sequence that is being aligned.
  152. However, the organism to which the aligned sequence belongs may not be present in the
  153. GFF file. Here you can specify the organism and feature type of the target sequences.
  154. The options here will apply to all targets unless the organism and type are explicity
  155. set in the GFF file using the 'target_organism' and 'target_type' attributes."),
  156. );
  157. $form['targets']['target_organism_id'] = array(
  158. '#title' => t('Target Organism'),
  159. '#type' => t('select'),
  160. '#description' => t("Optional. Choose the organism to which target sequences belong.
  161. Select this only if target sequences belong to a different organism than the
  162. one specified above. And only choose an organism here if all of the target sequences
  163. belong to the same species. If the targets in the GFF file belong to multiple
  164. different species then the organism must be specified using the 'target_organism=genus:species'
  165. attribute in the GFF file."),
  166. '#options' => $organisms,
  167. );
  168. $form['targets']['target_type'] = array(
  169. '#title' => t('Target Type'),
  170. '#type' => t('textfield'),
  171. '#description' => t("Optional. If the unique name for a target sequence is not unique (e.g. a protein
  172. and an mRNA have the same name) then you must specify the type for all targets in the GFF file. If
  173. the targets are of different types then the type must be specified using the 'target_type=type' attribute
  174. in the GFF file. This must be a valid Sequence Ontology (SO) term."),
  175. );
  176. $form['targets']['create_target']= array(
  177. '#type' => 'checkbox',
  178. '#title' => t('Create Target'),
  179. '#required' => FALSE,
  180. '#description' => t("If the target feature cannot be found, create one using the organism and type specified above, or
  181. using the 'target_organism' and 'target_type' fields specified in the GFF file. Values specified in the
  182. GFF file take precedence over those specified above."),
  183. );
  184. $form['button'] = array(
  185. '#type' => 'submit',
  186. '#value' => t('Import GFF3 file'),
  187. '#weight' => 10,
  188. );
  189. return $form;
  190. }
  191. /**
  192. *
  193. *
  194. * @ingroup gff3_loader
  195. */
  196. function tripal_feature_gff3_load_form_validate($form, &$form_state) {
  197. $gff_file = trim($form_state['values']['gff_file']);
  198. $organism_id = $form_state['values']['organism_id'];
  199. $target_organism_id = $form_state['values']['target_organism_id'];
  200. $target_type = trim($form_state['values']['target_type']);
  201. $create_target = $form_state['values']['create_target'];
  202. $create_organism = $form_state['values']['create_organism'];
  203. $add_only = $form_state['values']['add_only'];
  204. $update = $form_state['values']['update'];
  205. $refresh = $form_state['values']['refresh'];
  206. $remove = $form_state['values']['remove'];
  207. $use_transaction = $form_state['values']['use_transaction'];
  208. $line_number = trim($form_state['values']['line_number']);
  209. $landmark_type = trim($form_state['values']['landmark_type']);
  210. $alt_id_attr = trim($form_state['values']['alt_id_attr']);
  211. // check to see if the file is located local to Drupal
  212. $gff_file = trim($gff_file);
  213. $dfile = $_SERVER['DOCUMENT_ROOT'] . base_path() . $gff_file;
  214. if (!file_exists($dfile)) {
  215. // if not local to Drupal, the file must be someplace else, just use
  216. // the full path provided
  217. $dfile = $gff_file;
  218. }
  219. if (!file_exists($dfile)) {
  220. form_set_error('gff_file', t("Cannot find the file on the system. Check that the file exists or that the web server has permissions to read the file."));
  221. }
  222. // @coder-ignore: there are no functions being called here
  223. if (($add_only AND ($update OR $refresh OR $remove)) OR
  224. ($update AND ($add_only OR $refresh OR $remove)) OR
  225. ($refresh AND ($update OR $add_only OR $remove)) OR
  226. ($remove AND ($update OR $refresh OR $add_only))) {
  227. form_set_error('add_only', t("Please select only one checkbox from the import options section"));
  228. }
  229. if ($line_number and !is_numeric($line_number) or $line_number < 0) {
  230. form_set_error('line_number', t("Please provide an integer line number greater than zero."));
  231. }
  232. }
  233. /**
  234. *
  235. * @ingroup gff3_loader
  236. */
  237. function tripal_feature_gff3_load_form_submit($form, &$form_state) {
  238. global $user;
  239. $gff_file = trim($form_state['values']['gff_file']);
  240. $organism_id = $form_state['values']['organism_id'];
  241. $add_only = $form_state['values']['add_only'];
  242. $update = $form_state['values']['update'];
  243. $refresh = $form_state['values']['refresh'];
  244. $remove = $form_state['values']['remove'];
  245. $analysis_id = $form_state['values']['analysis_id'];
  246. $use_transaction = $form_state['values']['use_transaction'];
  247. $target_organism_id = $form_state['values']['target_organism_id'];
  248. $target_type = trim($form_state['values']['target_type']);
  249. $create_target = $form_state['values']['create_target'];
  250. $line_number = trim($form_state['values']['line_number']);
  251. $landmark_type = trim($form_state['values']['landmark_type']);
  252. $alt_id_attr = trim($form_state['values']['alt_id_attr']);
  253. $create_organism = $form_state['values']['create_organism'];
  254. $args = array($gff_file, $organism_id, $analysis_id, $add_only,
  255. $update, $refresh, $remove, $use_transaction, $target_organism_id,
  256. $target_type, $create_target, $line_number, $landmark_type, $alt_id_attr,
  257. $create_organism);
  258. $type = '';
  259. if ($add_only) {
  260. $type = 'import only new features';
  261. }
  262. if ($update) {
  263. $type = 'import all and update';
  264. }
  265. if ($refresh) {
  266. $type = 'import all and replace';
  267. }
  268. if ($remove) {
  269. $type = 'delete features';
  270. }
  271. $fname = preg_replace("/.*\/(.*)/", "$1", $gff_file);
  272. tripal_add_job("$type GFF3 file: $fname", 'tripal_feature',
  273. 'tripal_feature_load_gff3', $args, $user->uid);
  274. return '';
  275. }
  276. /**
  277. *
  278. *
  279. * @ingroup gff3_loader
  280. */
  281. function tripal_feature_load_gff3($gff_file, $organism_id, $analysis_id,
  282. $add_only = 0, $update = 0, $refresh = 0, $remove = 0, $use_transaction = 1,
  283. $target_organism_id = NULL, $target_type = NULL, $create_target = 0,
  284. $start_line = 1, $landmark_type = '', $alt_id_attr = '', $create_organism = FALSE,
  285. $job = NULL) {
  286. $ret = array();
  287. // empty the temp table
  288. $sql = "DELETE FROM {tripal_gff_temp}";
  289. chado_query($sql);
  290. // begin the transaction
  291. if ($use_transaction) {
  292. $transaction = tripal_db_start_transaction();
  293. print "\nNOTE: Loading of this GFF file is performed using a database transaction. \n" .
  294. "If the load fails or is terminated prematurely then the entire set of \n" .
  295. "insertions/updates is rolled back and will not be found in the database\n\n";
  296. }
  297. // check to see if the file is located local to Drupal
  298. $dfile = $_SERVER['DOCUMENT_ROOT'] . base_path() . $gff_file;
  299. if (!file_exists($dfile)) {
  300. // if not local to Drupal, the file must be someplace else, just use
  301. // the full path provided
  302. $dfile = $gff_file;
  303. }
  304. if (!file_exists($dfile)) {
  305. watchdog('T_gff3_loader', "Cannot find the file: %dfile",
  306. array('%dfile' => $dfile), WATCHDOG_ERROR);
  307. return 0;
  308. }
  309. print "Opening $gff_file\n";
  310. //$lines = file($dfile,FILE_SKIP_EMPTY_LINES);
  311. $fh = fopen($dfile, 'r');
  312. if (!$fh) {
  313. watchdog('T_gff3_loader', "cannot open file: %dfile",
  314. array('%dfile' => $dfile), WATCHDOG_ERROR);
  315. return 0;
  316. }
  317. $filesize = filesize($dfile);
  318. // get the controlled vocaubulary that we'll be using. The
  319. // default is the 'sequence' ontology
  320. $sql = "SELECT * FROM {cv} WHERE name = :cvname";
  321. $cv = chado_query($sql, array(':cvname' => 'sequence'))->fetchObject();
  322. if (!$cv) {
  323. watchdog('T_gff3_loader', "Cannot find the 'sequence' ontology",
  324. array(), WATCHDOG_ERROR);
  325. return '';
  326. }
  327. // get the organism for which this GFF3 file belongs
  328. $sql = "SELECT * FROM {organism} WHERE organism_id = :organism_id";
  329. $organism = chado_query($sql, array(':organism_id' => $organism_id))->fetchObject();
  330. $interval = intval($filesize * 0.0001);
  331. if ($interval == 0) {
  332. $interval = 1;
  333. }
  334. $in_fasta = 0;
  335. $line_num = 0;
  336. $num_read = 0;
  337. $intv_read = 0;
  338. // prepare the statement used to get the cvterm for each feature.
  339. $psql = "
  340. PREPARE sel_cvterm_idnasy (int, text, text) AS
  341. SELECT CVT.cvterm_id, CVT.cv_id, CVT.name, CVT.definition,
  342. CVT.dbxref_id, CVT.is_obsolete, CVT.is_relationshiptype
  343. FROM {cvterm} CVT
  344. INNER JOIN {cv} CV on CVT.cv_id = CV.cv_id
  345. LEFT JOIN {cvtermsynonym} CVTS on CVTS.cvterm_id = CVT.cvterm_id
  346. WHERE CV.cv_id = $1 and
  347. (lower(CVT.name) = lower($2) or lower(CVTS.synonym) = lower($3))
  348. ";
  349. $status = chado_query($psql);
  350. if (!$status) {
  351. watchdog('T_gff3_loader', 'cannot prepare statement \'sel_cvterm_idnasy\'.',
  352. array(), WATCHDOG_ERROR);
  353. return '';
  354. }
  355. // iterate through each line of the GFF file
  356. print "Parsing Line $line_num (0.00%). Memory: " . number_format(memory_get_usage()) . " bytes\r";
  357. while ($line = fgets($fh)) {
  358. $line_num++;
  359. $size = drupal_strlen($line);
  360. $num_read += $size;
  361. $intv_read += $size;
  362. if ($line_num < $start_line) {
  363. continue;
  364. }
  365. // update the job status every 1% features
  366. if ($job and $intv_read >= $interval) {
  367. $intv_read = 0;
  368. $percent = sprintf("%.2f", ($num_read / $filesize) * 100);
  369. print "Parsing Line $line_num (" . $percent . "%). Memory: " . number_format(memory_get_usage()) . " bytes.\r";
  370. tripal_job_set_progress($job, intval(($num_read / $filesize) * 100));
  371. }
  372. // check to see if we have FASTA section, if so then set the variable
  373. // to start parsing
  374. if (preg_match('/^##FASTA/i', $line)) {
  375. if ($remove) {
  376. // we're done because this is a delete operation so break out of the loop.
  377. break;
  378. }
  379. tripal_feature_load_gff3_fasta($fh, $interval, $num_read, $intv_read, $line_num);
  380. continue;
  381. }
  382. // if the ##sequence-region line is present then we want to add a new feature
  383. if (preg_match('/^##sequence-region (.*?) (\d+) (\d+)$/i', $line, $region_matches)) {
  384. $rid = $region_matches[1];
  385. $rstart = $region_matches[2];
  386. $rend = $region_matches[3];
  387. if ($landmark_type) {
  388. $result = chado_query("EXECUTE sel_cvterm_idnasy (:cv_id, :name, :synonym)",
  389. array(':cv_id' => $cv->cv_id, ':name' => $landmark_type, ':synonym' => $landmark_type));
  390. $cvterm = $result->fetchObject();
  391. if (!$cvterm) {
  392. watchdog('T_gff3_loader', 'cannot find feature type \'%landmark_type\' on line %line_num of the GFF file',
  393. array('%landmark_type' => $landmark_type, '%line_num' => $line_num), WATCHDOG_ERROR);
  394. return '';
  395. }
  396. tripal_feature_load_gff3_feature($organism, $analysis_id, $cvterm, $rid,
  397. $rid, '', 'f', 'f', 1, 0);
  398. }
  399. continue;
  400. }
  401. // skip comments
  402. if (preg_match('/^#/', $line)) {
  403. continue;
  404. }
  405. // skip empty lines
  406. if (preg_match('/^\s*$/', $line)) {
  407. continue;
  408. }
  409. // get the columns
  410. $cols = explode("\t", $line);
  411. if (sizeof($cols) != 9) {
  412. watchdog('T_gff3_loader', 'improper number of columns on line %line_num',
  413. array('%line_num' => $line_num), WATCHDOG_ERROR);
  414. return '';
  415. }
  416. // get the column values
  417. $landmark = $cols[0];
  418. $source = $cols[1];
  419. $type = $cols[2];
  420. $start = $cols[3];
  421. $end = $cols[4];
  422. $score = $cols[5];
  423. $strand = $cols[6];
  424. $phase = $cols[7];
  425. $attrs = explode(";", $cols[8]); // split by a semicolon
  426. // ready the start and stop for chado. Chado expects these positions
  427. // to be zero-based, so we substract 1 from the fmin
  428. $fmin = $start - 1;
  429. $fmax = $end;
  430. if ($end < $start) {
  431. $fmin = $end - 1;
  432. $fmax = $start;
  433. }
  434. // format the strand for chado
  435. if (strcmp($strand, '.') == 0) {
  436. $strand = 0;
  437. }
  438. elseif (strcmp($strand, '+') == 0) {
  439. $strand = 1;
  440. }
  441. elseif (strcmp($strand, '-') == 0) {
  442. $strand = -1;
  443. }
  444. if (strcmp($phase, '.') == 0) {
  445. $phase = '';
  446. }
  447. $result = chado_query("EXECUTE sel_cvterm_idnasy (:cv_id, :name, :synonym)",
  448. array(':cv_id' => $cv->cv_id, ':name' => $type, ':synonym' => $type));
  449. $cvterm = $result->fetchObject();
  450. if (!$cvterm) {
  451. watchdog('T_gff3_loader', 'cannot find feature term \'%type\' on line %line_num of the GFF file',
  452. array('%type' => $type, '%line_num' => $line_num), WATCHDOG_ERROR);
  453. return '';
  454. }
  455. // break apart each of the attributes
  456. $tags = array();
  457. $attr_name = '';
  458. $attr_uniquename = '';
  459. $attr_residue_info = '';
  460. $attr_locgroup = 0;
  461. $attr_fmin_partial = 'f';
  462. $attr_fmax_partial = 'f';
  463. $attr_is_obsolete = 'f';
  464. $attr_is_analysis = 'f';
  465. $attr_others = '';
  466. $residues = '';
  467. // the organism to which a feature belongs can be set in the GFF
  468. // file using the 'organism' attribute. By default we
  469. // set the $feature_organism variable to the default organism for the landmark
  470. $attr_organism = '';
  471. $feature_organism = $organism;
  472. foreach ($attrs as $attr) {
  473. $attr = rtrim($attr);
  474. $attr = ltrim($attr);
  475. if (strcmp($attr, '')==0) {
  476. continue;
  477. }
  478. if (!preg_match('/^[^\=]+\=.+$/', $attr)) {
  479. watchdog('T_gff3_loader', 'Attribute is not correctly formatted on line %line_num: %attr',
  480. array('%line_num' => $line_num, '%attr' => $attr), WATCHDOG_ERROR);
  481. return '';
  482. }
  483. // break apart each tag
  484. $tag = preg_split("/=/", $attr, 2); // split by equals sign
  485. // multiple instances of an attribute are separated by commas
  486. $tag_name = $tag[0];
  487. if (!array_key_exists($tag_name, $tags)) {
  488. $tags[$tag_name] = array();
  489. }
  490. $tags[$tag_name] = array_merge($tags[$tag_name], explode(",", $tag[1])); // split by comma
  491. // replace the URL escape codes for each tag
  492. for ($i = 0; $i < count($tags[$tag_name]); $i++) {
  493. $tags[$tag_name][$i] = urldecode($tags[$tag_name][$i]);
  494. }
  495. // get the name and ID tags
  496. $skip_feature = 0; // if there is a problem with any of the attributes this variable gets set
  497. if (strcmp($tag_name, 'ID') == 0) {
  498. $attr_uniquename = urldecode($tag[1]);
  499. }
  500. elseif (strcmp($tag_name, 'Name') == 0) {
  501. $attr_name = urldecode($tag[1]);
  502. }
  503. elseif (strcmp($tag_name, 'organism') == 0) {
  504. $attr_organism = urldecode($tag[1]);
  505. $org_matches = array();
  506. if (preg_match('/^(.*?):(.*?)$/', $attr_organism, $org_matches)) {
  507. $values = array(
  508. 'genus' => $org_matches[1],
  509. 'species' => $org_matches[2],
  510. );
  511. $options = array('statement_name' => 'sel_organism_gesp');
  512. $org = tripal_core_chado_select('organism', array("*"), $values, $options);
  513. if (count($org) == 0) {
  514. if ($create_organism) {
  515. $options = array('statement_name' => 'ins_organism_gesp');
  516. $feature_organism = (object) tripal_core_chado_insert('organism', $values, $options);
  517. if (!$feature_organism) {
  518. watchdog('T_gff3_loader', "Could not add the organism, '%org', from line %line. Skipping this line. ",
  519. array('%org' => $attr_organism, '%line' => $line_num), WATCHDOG_ERROR);
  520. $skip_feature = 1;
  521. }
  522. }
  523. else {
  524. watchdog('T_gff3_loader', "The organism attribute '%org' on line %line does not exist. Skipping this line. ",
  525. array('%org' => $attr_organism, '%line' => $line_num), WATCHDOG_ERROR);
  526. $skip_feature = 1;
  527. }
  528. }
  529. else {
  530. // we found the organism in the database so use it
  531. $feature_organism = $org[0];
  532. }
  533. }
  534. else {
  535. watchdog('T_gff3_loader', "The organism attribute '%org' on line %line is not properly formated. It " .
  536. "should be of the form: organism=Genus:species. Skipping this line.",
  537. array('%org' => $attr_organism, '%line' => $line_num), WATCHDOG_ERROR);
  538. $skip_feature = 1;
  539. }
  540. }
  541. // get the list of non-reserved attributes
  542. elseif (strcmp($tag_name, 'Alias') != 0 and strcmp($tag_name, 'Parent') != 0 and
  543. strcmp($tag_name, 'Target') != 0 and strcmp($tag_name, 'Gap') != 0 and
  544. strcmp($tag_name, 'Derives_from') != 0 and strcmp($tag_name, 'Note') != 0 and
  545. strcmp($tag_name, 'Dbxref') != 0 and strcmp($tag_name, 'Ontology_term') != 0 and
  546. strcmp($tag_name, 'Is_circular') != 0 and strcmp($tag_name, 'target_organism') != 0 and
  547. strcmp($tag_name, 'target_type') != 0 and strcmp($tag_name, 'organism' != 0)) {
  548. foreach ($tags[$tag_name] as $value) {
  549. $attr_others[$tag_name][] = $value;
  550. }
  551. }
  552. }
  553. if ($skip_line) {
  554. continue;
  555. }
  556. // if neither name nor uniquename are provided then generate one
  557. if (!$attr_uniquename and !$attr_name) {
  558. // check if an alternate ID field is suggested, if so, then use
  559. // that for the name
  560. if (array_key_exists($alt_id_attr, $tags)) {
  561. $attr_uniquename = $tags[$alt_id_attr][0];
  562. $attr_name = $attr_uniquename;
  563. }
  564. // if the row has a parent then generate a uniquename using the parent name
  565. // add the date to the name in the event there are more than one child with
  566. // the same parent.
  567. elseif (array_key_exists('Parent', $tags)) {
  568. $date = getdate();
  569. $attr_uniquename = $tags['Parent'][0] . "-$type-$landmark-" . $date[0] . ":" . ($fmin + 1) . ".." . $fmax;
  570. $attr_name = $attr_uniquename;
  571. }
  572. // generate a unique name based on the date, type and location
  573. // and set the name to simply be the type
  574. else {
  575. $date = getdate();
  576. $attr_uniquename = $date[0] . "-$type-$landmark:" . ($fmin + 1) . ".." . $fmax;
  577. $attr_name = $type;
  578. }
  579. }
  580. // if a name is not specified then use the unique name as the name
  581. if (strcmp($attr_name, '')==0) {
  582. $attr_name = $attr_uniquename;
  583. }
  584. // if an ID attribute is not specified then use the attribute name plus the date
  585. if (!$attr_uniquename) {
  586. $date = getdate();
  587. $attr_uniquename = $attr_name . '-' . $date[0];
  588. }
  589. // make sure the landmark sequence exists in the database. If the user
  590. // has not specified a landmark type (and it's not requiredin the GFF foramt)
  591. // then We don't know the type of the landmark so we'll hope that it's unique across
  592. // all types for the orgnaism. Only do this test if the landmark and the feature are
  593. // different.
  594. if (!$remove and !(strcmp($landmark, $attr_uniquename) == 0 or strcmp($landmark, $attr_name) == 0)) {
  595. $select = array(
  596. 'organism_id' => $organism->organism_id,
  597. 'uniquename' => $landmark,
  598. );
  599. $columns = array('count(*) as num_landmarks');
  600. $options = array('statement_name' => 'sel_feature_numland');
  601. if ($landmark_type) {
  602. $select['type_id'] = array(
  603. 'name' => $landmark_type,
  604. );
  605. $options = array('statement_name' => 'sel_feature_numlandty');
  606. }
  607. $count = tripal_core_chado_select('feature', $columns, $select, $options);
  608. if (!$count or count($count) == 0 or $count[0]->num_landmarks == 0) {
  609. // now look for the landmark using the name rather than uniquename.
  610. $select = array(
  611. 'organism_id' => $organism->organism_id,
  612. 'name' => $landmark,
  613. );
  614. $columns = array('count(*) as num_landmarks');
  615. $options = array('statement_name' => 'sel_feature_numlandna');
  616. if ($landmark_type) {
  617. $select['type_id'] = array(
  618. 'name' => $landmark_type,
  619. );
  620. $options = array('statement_name' => 'sel_feature_numlandnaty');
  621. }
  622. $count = tripal_core_chado_select('feature', $columns, $select, $options);
  623. if (!$count or count($count) == 0 or $count[0]->num_landmarks == 0) {
  624. watchdog('T_gff3_loader', "The landmark '%landmark' cannot be found for this organism (%species) " .
  625. "Please add the landmark and then retry the import of this GFF3 " .
  626. "file", array('%landmark' => $landmark, '%species' => $organism->genus . " " . $organism->species), WATCHDOG_ERROR);
  627. return '';
  628. }
  629. elseif ($count[0]->num_landmarks > 1) {
  630. watchdog('T_gff3_loader', "The landmark '%landmark' has more than one entry for this organism (%species) " .
  631. "Cannot continue", array('%landmark' => $landmark, '%species' => $organism->genus . " " . $organism->species), WATCHDOG_ERROR);
  632. return '';
  633. }
  634. }
  635. if ($count[0]->num_landmarks > 1) {
  636. watchdog('T_gff3_loader', "The landmark '%landmark' is not unique for this organism. " .
  637. "The features cannot be associated", array('%landmark' => $landmark), WATCHDOG_ERROR);
  638. return '';
  639. }
  640. }
  641. // if the option is to remove or refresh then we want to remove
  642. // the feature from the database.
  643. if ($remove or $refresh) {
  644. $sql = "DELETE FROM {feature}
  645. WHERE organism_id = %d and uniquename = '%s' and type_id = %d";
  646. $match = array(
  647. 'organism_id' => $feature_organism->organism_id,
  648. 'uniquename' => $attr_uniquename,
  649. 'type_id' => $cvterm->cvterm_id
  650. );
  651. $result = tripal_core_chado_delete('feature', $match);
  652. if (!$result) {
  653. watchdog('T_gff3_loader', "cannot delete feature %attr_uniquename",
  654. array('%attr_uniquename' => $attr_uniquename), WATCHDOG_ERROR);
  655. }
  656. $feature = 0;
  657. unset($result);
  658. }
  659. // add or update the feature and all properties
  660. if ($update or $refresh or $add_only) {
  661. // add/update the feature
  662. $feature = tripal_feature_load_gff3_feature($feature_organism, $analysis_id, $cvterm,
  663. $attr_uniquename, $attr_name, $residues, $attr_is_analysis,
  664. $attr_is_obsolete, $add_only, $score);
  665. if ($feature) {
  666. // add a record for this feature to the tripal_gff_temp table for
  667. // later lookup
  668. $values = array(
  669. 'feature_id' => $feature->feature_id,
  670. 'organism_id' => $feature->organism_id,
  671. 'type_name' => $type,
  672. 'uniquename' => $feature->uniquename
  673. );
  674. // make sure this record doesn't already exist in oru temp table
  675. $options = array('statement_name' => 'sel_tripalgfftemp_all');
  676. $results = tripal_core_chado_select('tripal_gff_temp', array('*'), $values, $options);
  677. if (count($results) == 0) {
  678. $options = array('statement_name' => 'ins_tripalgfftemp');
  679. $result = tripal_core_chado_insert('tripal_gff_temp', $values, $options);
  680. if (!$result) {
  681. watchdog('T_gff3_loader', "Cound not save record in temporary table, Cannot continue.", array(), WATCHDOG_ERROR);
  682. exit;
  683. }
  684. }
  685. // add/update the featureloc if the landmark and the ID are not the same
  686. // if they are the same then this entry in the GFF is probably a landmark identifier
  687. if (strcmp($landmark, $attr_uniquename) !=0 ) {
  688. tripal_feature_load_gff3_featureloc($feature, $organism,
  689. $landmark, $fmin, $fmax, $strand, $phase, $attr_fmin_partial,
  690. $attr_fmax_partial, $attr_residue_info, $attr_locgroup);
  691. }
  692. continue;
  693. // add any aliases for this feature
  694. if (array_key_exists('Alias', $tags)) {
  695. tripal_feature_load_gff3_alias($feature, $tags['Alias']);
  696. }
  697. // add any dbxrefs for this feature
  698. if (array_key_exists('Dbxref', $tags)) {
  699. tripal_feature_load_gff3_dbxref($feature, $tags['Dbxref']);
  700. }
  701. // add any ontology terms for this feature
  702. if (array_key_exists('Ontology_term', $tags)) {
  703. tripal_feature_load_gff3_ontology($feature, $tags['Ontology_term']);
  704. }
  705. // add parent relationships
  706. if (array_key_exists('Parent', $tags)) {
  707. tripal_feature_load_gff3_parents($feature, $cvterm, $tags['Parent'], $feature_organism->organism_id, $fmin);
  708. }
  709. // add target relationships
  710. if (array_key_exists('Target', $tags)) {
  711. tripal_feature_load_gff3_target($feature, $tags, $target_organism_id, $target_type, $create_target, $attr_locgroup);
  712. }
  713. // add gap information. This goes in simply as a property
  714. if (array_key_exists('Gap', $tags)) {
  715. foreach ($tags['Gap'] as $value) {
  716. tripal_feature_load_gff3_property($feature, 'Gap', $value);
  717. }
  718. }
  719. // add notes. This goes in simply as a property
  720. if (array_key_exists('Note', $tags)) {
  721. foreach ($tags['Note'] as $value) {
  722. tripal_feature_load_gff3_property($feature, 'Note', $value);
  723. }
  724. }
  725. // add the Derives_from relationship (e.g. polycistronic genes).
  726. if (array_key_exists('Derives_from', $tags)) {
  727. tripal_feature_load_gff3_derives_from($feature, $tags['Derives_from'][0], $feature_organism);
  728. }
  729. // add in the GFF3_source dbxref so that GBrowse can find the feature using the source column
  730. $source_ref = array('GFF_source:' . $source);
  731. tripal_feature_load_gff3_dbxref($feature, $source_ref);
  732. // add any additional attributes
  733. if ($attr_others) {
  734. foreach ($attr_others as $tag_name => $values) {
  735. foreach ($values as $value) {
  736. tripal_feature_load_gff3_property($feature, $tag_name, $value);
  737. }
  738. }
  739. }
  740. }
  741. }
  742. }
  743. if (!$remove) {
  744. print "\nSetting ranks of children...\n";
  745. // get features in a relationship that are also children of an alignment
  746. $sql = "
  747. SELECT DISTINCT F.feature_id, F.organism_id, F.type_id,
  748. F.uniquename, FL.strand
  749. FROM {tripal_gff_temp} TGT
  750. INNER JOIN {feature} F ON TGT.feature_id = F.feature_id
  751. INNER JOIN {feature_relationship} FR ON FR.object_id = TGT.feature_id
  752. INNER JOIN {cvterm} CVT ON CVT.cvterm_id = FR.type_id
  753. INNER JOIN {featureloc} FL ON FL.feature_id = F.feature_id
  754. WHERE CVT.name = 'part_of'
  755. ";
  756. $parents = chado_query($sql);
  757. // build and prepare the SQL for selecting the children relationship
  758. $sql = "";
  759. $sql = "
  760. PREPARE sel_gffchildren (int) AS
  761. SELECT DISTINCT FR.feature_relationship_id, FL.fmin, FR.rank
  762. FROM {feature_relationship} FR
  763. INNER JOIN {featureloc} FL on FL.feature_id = FR.subject_id
  764. WHERE FR.object_id = \$1 ORDER BY FL.fmin ASC
  765. ";
  766. $success = chado_query('sel_gffchildren', $sql);
  767. if (!$success) {
  768. watchdog("T_gff3_loader", "Cannot prepare statement 'sel_gffchildren' and cannot set children ranks.",
  769. array(), WATCHDOG_WARNING);
  770. return 0;
  771. }
  772. // now set the rank of any parent/child relationships. The order is based
  773. // on the fmin. The start rank is 1. This allows features with other
  774. // relationships to be '0' (the default), and doesn't interfer with the
  775. // ordering defined here.
  776. while ($parent = $parents->fetchObject()) {
  777. // get the children
  778. if ($connection) {
  779. $result = chado_query('EXECUTE sel_gffchildren (:feature_id)', array(':feature_id' => $parent->feature_id));
  780. }
  781. else {
  782. $result = chado_query($sql, array(':feature_id' => $parent->feature_id));
  783. }
  784. // build an array of the children
  785. $children = array();
  786. while ($child = $result->fetchObject()) {
  787. $children[] = $child;
  788. }
  789. // the children list comes sorted in ascending fmin
  790. // but if the parent is on the reverse strand we need to
  791. // reverse the order of the children.
  792. if ($parent->strand == -1) {
  793. arsort($children);
  794. }
  795. // first set the ranks to a negative number so that we don't
  796. // get a duplicate error message when we try to change any of them
  797. $rank = -1;
  798. foreach ($children as $child) {
  799. $match = array('feature_relationship_id' => $child->feature_relationship_id);
  800. $options = array('statement_name' => 'upd_featurerelationship_rank');
  801. $values = array('rank' => $rank);
  802. tripal_core_chado_update('feature_relationship', $match, $values, $options);
  803. $rank--;
  804. }
  805. // now set the rank correctly. The rank should start at 0.
  806. $rank = 0;
  807. foreach ($children as $child) {
  808. $match = array('feature_relationship_id' => $child->feature_relationship_id);
  809. $options = array('statement_name' => 'upd_featurerelationship_rank');
  810. $values = array('rank' => $rank);
  811. //print "Was: " . $child->rank . " now $rank ($parent->strand)\n" ;
  812. tripal_core_chado_update('feature_relationship', $match, $values, $options);
  813. $rank++;
  814. }
  815. }
  816. }
  817. // commit the transaction
  818. if ($use_transaction) {
  819. tripal_db_commit_transaction();
  820. }
  821. print "Done\n";
  822. return 1;
  823. }
  824. /**
  825. *
  826. *
  827. * @ingroup gff3_loader
  828. */
  829. function tripal_feature_load_gff3_derives_from($feature, $subject, $organism) {
  830. // get the subject type
  831. $values = array(
  832. 'organism_id' => $organism->organism_id,
  833. 'uniquename' => $subject,
  834. );
  835. $options = array('statement_name' => 'sel_tripalgfftemp_orun');
  836. $result = tripal_core_chado_select('tripal_gff_temp', array('type_name'), $values, $options);
  837. if (count($result) == 0) {
  838. watchdog("T_gff3_loader", "Cannot find subject type for feature in 'derives_from' relationship: %subject", array('%subject' => $subject), WATCHDOG_WARNING);
  839. return '';
  840. }
  841. $subject_type = $result[0]->type_name;
  842. // get the subject feature
  843. $match = array(
  844. 'organism_id' => $organism->organism_id,
  845. 'uniquename' => $subject,
  846. 'type_id' => array(
  847. 'name' => $subject_type,
  848. 'cv_id' => array(
  849. 'name' => 'sequence'
  850. ),
  851. ),
  852. );
  853. $options = array('statement_name' => 'sel_feature_orunty');
  854. $sfeature = tripal_core_chado_select('feature', array('feature_id'), $match, $options);
  855. if (count($sfeature)==0) {
  856. watchdog('T_gff3_loader', "Could not add 'Derives_from' relationship " .
  857. "for %uniquename and %subject. Subject feature, '%subject', " .
  858. "cannot be found", array('%uniquename' => $feature->uniquename, '%subject' => $subject), WATCHDOG_ERROR);
  859. return;
  860. }
  861. // now check to see if the relationship already exists
  862. $values = array(
  863. 'object_id' => $sfeature[0]->feature_id,
  864. 'subject_id' => $feature->feature_id,
  865. 'type_id' => array(
  866. 'cv_id' => array(
  867. 'name' => 'relationship'
  868. ),
  869. 'name' => 'derives_from',
  870. ),
  871. 'rank' => 0
  872. );
  873. $options = array('statement_name' => 'sel_featurerelationship_objectid_subjectid_typeid_rank');
  874. $rel = tripal_core_chado_select('feature_relationship', array('*'), $values, $options);
  875. if (count($rel) > 0) {
  876. return;
  877. }
  878. // finally insert the relationship if it doesn't exist
  879. $options = array('statement_name' => 'ins_featurerelationship_objectid_subjectid_typeid_rank');
  880. $ret = tripal_core_chado_insert('feature_relationship', $values, $options);
  881. if (!$ret) {
  882. watchdog("T_gff3_loader", "Could not add 'Derives_from' relationship for $feature->uniquename and $subject",
  883. array(), WATCHDOG_WARNING);
  884. }
  885. }
  886. /**
  887. *
  888. *
  889. * @ingroup gff3_loader
  890. */
  891. function tripal_feature_load_gff3_parents($feature, $cvterm, $parents, $organism_id, $fmin) {
  892. $uname = $feature->uniquename;
  893. $type = $cvterm->name;
  894. $rel_type = 'part_of';
  895. // prepare these SQL statements that will be used repeatedly.
  896. $psql = "
  897. PREPARE sel_cvterm_cvname_cvtname_synonym (text, text, text) AS
  898. SELECT CVT.cvterm_id
  899. FROM {cvterm} CVT
  900. INNER JOIN {cv} CV on CVT.cv_id = CV.cv_id
  901. LEFT JOIN {cvtermsynonym} CVTS on CVTS.cvterm_id = CVT.cvterm_id
  902. WHERE cv.name = $1 and (CVT.name = $2 or CVTS.synonym = $3)
  903. ";
  904. $status = chado_query($psql);
  905. if (!$status) {
  906. watchdog("T_gff3_loader", "Cannot prepare statement 'sel_cvterm_cvname_cvtname_synonym' for ontology term",
  907. array(), WATCHDOG_WARNING);
  908. return '';
  909. }
  910. // iterate through the parents in the list
  911. foreach ($parents as $parent) {
  912. // get the parent cvterm
  913. $values = array(
  914. 'organism_id' => $organism_id,
  915. 'uniquename' => $parent,
  916. );
  917. $options = array('statement_name' => 'sel_tripalgfftemp_orun');
  918. $result = tripal_core_chado_select('tripal_gff_temp', array('type_name'), $values, $options);
  919. if (count($result) == 0) {
  920. watchdog("T_gff3_loader", "Cannot find parent: %parent", array('%parent' => $parent), WATCHDOG_WARNING);
  921. return '';
  922. }
  923. $parent_type = $result[0]->type_name;
  924. // try to find the parent
  925. $parentcvterm = chado_query("EXECUTE sel_cvterm_cvname_cvtname_synonym (:cvname, :name, :synonym)",
  926. array(':cvname' => 'sequence', ':name' => $parent_type, ':synonym' => $parent_type))->fetchObject();
  927. $relcvterm = chado_query("EXECUTE sel_cvterm_cvname_cvtname_synonym (:cvname, :name, :synonym)",
  928. array(':cvname' => 'relationship', ':name' => $rel_type, ':synonymn' => $rel_type))->fetchObject();
  929. $values = array(
  930. 'organism_id' => $organism_id,
  931. 'uniquename' => $parent,
  932. 'type_id' => $parentcvterm->cvterm_id,
  933. );
  934. $options = array('statement_name' => 'sel_feature_orunty');
  935. $result = tripal_core_chado_select('feature', array('feature_id'), $values, $options);
  936. $parent_feature = $result[0];
  937. // if the parent exists then add the relationship otherwise print error and skip
  938. if ($parent_feature) {
  939. // check to see if the relationship already exists
  940. $values = array(
  941. 'object_id' => $parent_feature->feature_id,
  942. 'subject_id' => $feature->feature_id,
  943. 'type_id' => $relcvterm->cvterm_id,
  944. );
  945. $options = array('statement_name' => 'sel_featurerelationship_objectid_subjectid_typeid');
  946. $rel = tripal_core_chado_select('feature_relationship', array('*'), $values, $options);
  947. if (count($rel) > 0) {
  948. }
  949. else {
  950. // the relationship doesn't already exist, so add it.
  951. $values = array(
  952. 'subject_id' => $feature->feature_id,
  953. 'object_id' => $parent_feature->feature_id,
  954. 'type_id' => $relcvterm->cvterm_id,
  955. );
  956. $options = array('statement_name' => 'ins_featurerelationship_subjectid_objectid_typeid');
  957. $result = tripal_core_chado_insert('feature_relationship', $values, $options);
  958. if (!$result) {
  959. watchdog("T_gff3_loader", "Failed to insert feature relationship '$uname' ($type) $rel_type '$parent' ($parent_type)",
  960. array(), WATCHDOG_WARNING);
  961. }
  962. }
  963. }
  964. else {
  965. watchdog("T_gff3_loader", "Cannot establish relationship '$uname' ($type) $rel_type '$parent' ($parent_type): Cannot find the parent",
  966. array(), WATCHDOG_WARNING);
  967. }
  968. }
  969. }
  970. /**
  971. *
  972. *
  973. * @ingroup gff3_loader
  974. */
  975. function tripal_feature_load_gff3_dbxref($feature, $dbxrefs) {
  976. // iterate through each of the dbxrefs
  977. foreach ($dbxrefs as $dbxref) {
  978. // get the database name from the reference. If it doesn't exist then create one.
  979. $ref = explode(":", $dbxref);
  980. $dbname = $ref[0];
  981. $accession = $ref[1];
  982. // first look for the database name if it doesn't exist then create one.
  983. // first check for the fully qualified URI (e.g. DB:<dbname>. If that
  984. // can't be found then look for the name as is. If it still can't be found
  985. // the create the database
  986. $values = array('name' => "DB:$dbname");
  987. $options = array('statement_name' => 'sel_db_name');
  988. $db = tripal_core_chado_select('db', array('db_id'), $values, $options);
  989. if (count($db) == 0) {
  990. $values = array('name' => "$dbname");
  991. $db = tripal_core_chado_select('db', array('db_id'), $values, $options);
  992. }
  993. if (count($db) == 0) {
  994. $values = array(
  995. 'name' => $dbname,
  996. 'description' => 'Added automatically by the GFF loader'
  997. );
  998. $options = array('statement_name' => 'ins_db_name');
  999. $success = tripal_core_chado_insert('db', $values, $options);
  1000. if ($success) {
  1001. $values = array('name' => "$dbname");
  1002. $options = array('statement_name' => 'sel_db_name');
  1003. $db = tripal_core_chado_select('db', array('db_id'), $values, $options);
  1004. }
  1005. else {
  1006. watchdog("T_gff3_loader", "Cannot find or add the database $dbname", array(), WATCHDOG_WARNING);
  1007. return 0;
  1008. }
  1009. }
  1010. $db = $db[0];
  1011. // now check to see if the accession exists
  1012. $values = array(
  1013. 'accession' => $accession,
  1014. 'db_id' => $db->db_id
  1015. );
  1016. $options = array('statement_name' => 'sel_dbxref_accession_dbid');
  1017. $dbxref = tripal_core_chado_select('dbxref', array('dbxref_id'), $values, $options);
  1018. // if the accession doesn't exist then we want to add it
  1019. if (sizeof($dbxref) == 0) {
  1020. $values = array(
  1021. 'db_id' => $db->db_id,
  1022. 'accession' => $accession,
  1023. 'version' => ''
  1024. );
  1025. $options = array('statement_name' => 'ins_dbxref_dbid_accession_version');
  1026. $ret = tripal_core_chado_insert('dbxref', $values, $options);
  1027. $values = array(
  1028. 'accession' => $accession,
  1029. 'db_id' => $db->db_id
  1030. );
  1031. $options = array('statement_name' => 'sel_dbxref_accession_dbid');
  1032. $dbxref = tripal_core_chado_select('dbxref', array('dbxref_id'), $values, $options);
  1033. }
  1034. $dbxref = $dbxref[0];
  1035. // check to see if this feature dbxref already exists
  1036. $values = array(
  1037. 'dbxref_id' => $dbxref->dbxref_id,
  1038. 'feature_id' => $feature->feature_id
  1039. );
  1040. $options = array('statement_name' => 'sel_featuredbxref_dbxrefid_featureid');
  1041. $fdbx = tripal_core_chado_select('feature_dbxref', array('feature_dbxref_id'), $values, $options);
  1042. // now associate this feature with the database reference if it doesn't
  1043. // already exist
  1044. if (sizeof($fdbx) == 0) {
  1045. $values = array(
  1046. 'dbxref_id' => $dbxref->dbxref_id,
  1047. 'feature_id' => $feature->feature_id
  1048. );
  1049. $options = array('statement_name' => 'ins_featuredbxref_dbxrefid_featureid');
  1050. $success = tripal_core_chado_insert('feature_dbxref', $values, $options);
  1051. if (!$success) {
  1052. watchdog("T_gff3_loader", "Failed to insert Dbxref: $dbname:$accession", array(), WATCHDOG_WARNING);
  1053. return 0;
  1054. }
  1055. }
  1056. }
  1057. return 1;
  1058. }
  1059. /**
  1060. *
  1061. *
  1062. * @ingroup gff3_loader
  1063. */
  1064. function tripal_feature_load_gff3_ontology($feature, $dbxrefs) {
  1065. // iterate through each of the dbxrefs
  1066. foreach ($dbxrefs as $dbxref) {
  1067. // get the database name from the reference. If it doesn't exist then create one.
  1068. $ref = explode(":", $dbxref);
  1069. $dbname = $ref[0];
  1070. $accession = $ref[1];
  1071. // first look for the database name
  1072. $options = array('statement_name' => 'sel_db_name');
  1073. $db = tripal_core_chado_select('db', array('db_id'), array('name' => "DB:$dbname"), $options);
  1074. if (sizeof($db) == 0) {
  1075. // now look for the name without the 'DB:' prefix.
  1076. $db = tripal_core_chado_select('db', array('db_id'), array('name' => "$dbname"), $options);
  1077. if (sizeof($db) == 0) {
  1078. watchdog("T_gff3_loader", "Database, $dbname, is not present. Cannot associate term: $dbname:$accession", array(), WATCHDOG_WARNING);
  1079. return 0;
  1080. }
  1081. }
  1082. $db = $db[0];
  1083. // now check to see if the accession exists
  1084. $options = array('statement_name' => 'sel_dbxref_accession_dbid');
  1085. $dbxref = tripal_core_chado_select('dbxref', array('dbxref_id'),
  1086. array('accession' => $accession, 'db_id' => $db->db_id), $options);
  1087. if (sizeof($dbxref) == 0) {
  1088. watchdog("T_gff3_loader", "Accession, $accession is missing for reference: $dbname:$accession", array(), WATCHDOG_WARNING);
  1089. return 0;
  1090. }
  1091. $dbxref = $dbxref[0];
  1092. // now check to see if the cvterm exists
  1093. $options = array('statement_name' => 'sel_cvterm_dbxrefid');
  1094. $cvterm = tripal_core_chado_select('cvterm', array('cvterm_id'), array(
  1095. 'dbxref_id' => $dbxref->dbxref_id), $options);
  1096. // if it doesn't exist in the cvterm table, look for an alternate id
  1097. if (sizeof($cvterm) == 0) {
  1098. $options = array('statement_name' => 'sel_cvtermdbxref_dbxrefid');
  1099. $cvterm = tripal_core_chado_select('cvterm_dbxref', array('cvterm_id'), array(
  1100. 'dbxref_id' => $dbxref->dbxref_id), $options);
  1101. if (sizeof($cvterm) == 0) {
  1102. watchdog("T_gff3_loader", "CV Term is missing for reference: $dbname:$accession", array(), WATCHDOG_WARNING);
  1103. return 0;
  1104. }
  1105. }
  1106. $cvterm = $cvterm[0];
  1107. // check to see if this feature cvterm already exists
  1108. $options = array('statement_name' => 'sel_featurecvterm_cvtermid_featureid');
  1109. $fcvt = tripal_core_chado_select('feature_cvterm', array('feature_cvterm_id'),
  1110. array('cvterm_id' => $cvterm->cvterm_id, 'feature_id' => $feature->feature_id),
  1111. $options);
  1112. // now associate this feature with the cvterm if it doesn't already exist
  1113. if (sizeof($fcvt)==0) {
  1114. $values = array(
  1115. 'cvterm_id' => $cvterm->cvterm_id,
  1116. 'feature_id' => $feature->feature_id,
  1117. 'pub_id' => array(
  1118. 'uniquename' => 'null',
  1119. ),
  1120. );
  1121. $options = array('statement_name' => 'ins_featurecvterm_cvtermid_featureid_pubid');
  1122. $success = tripal_core_chado_insert('feature_cvterm', $values, $options);
  1123. if (!$success) {
  1124. watchdog("T_gff3_loader", "Failed to insert ontology term: $dbname:$accession", array(), WATCHDOG_WARNING);
  1125. return 0;
  1126. }
  1127. }
  1128. }
  1129. return 1;
  1130. }
  1131. /**
  1132. *
  1133. *
  1134. * @ingroup gff3_loader
  1135. */
  1136. function tripal_feature_load_gff3_alias($feature, $aliases) {
  1137. // make sure we have a 'synonym_type' vocabulary
  1138. $select = array('name' => 'synonym_type');
  1139. $options = array('statement_name' => 'sel_cv_name');
  1140. $results = tripal_core_chado_select('cv', array('*'), $select, $options);
  1141. if (count($results) == 0) {
  1142. // insert the 'synonym_type' vocabulary
  1143. $values = array(
  1144. 'name' => 'synonym_type',
  1145. 'definition' => 'vocabulary for synonym types',
  1146. );
  1147. $options = array('statement_name' => 'ins_cv_name_definition');
  1148. $success = tripal_core_chado_insert('cv', $values, $options);
  1149. if (!$success) {
  1150. watchdog("T_gff3_loader", "Failed to add the synonyms type vocabulary", array(), WATCHDOG_WARNING);
  1151. return 0;
  1152. }
  1153. // now that we've added the cv we need to get the record
  1154. $options = array('statement_name' => 'sel_cv_name');
  1155. $results = tripal_core_chado_select('cv', array('*'), $select, $options);
  1156. if (count($results) > 0) {
  1157. $syncv = $results[0];
  1158. }
  1159. }
  1160. else {
  1161. $syncv = $results[0];
  1162. }
  1163. // get the 'exact' cvterm, which is the type of synonym we're adding
  1164. $select = array(
  1165. 'name' => 'exact',
  1166. 'cv_id' => array(
  1167. 'name' => 'synonym_type'
  1168. ),
  1169. );
  1170. $options = array('statement_name' => 'sel_cvterm_name_cvid');
  1171. $result = tripal_core_chado_select('cvterm', array('*'), $select, $options);
  1172. if (count($result) == 0) {
  1173. $term = array(
  1174. 'name' => 'exact',
  1175. 'id' => "internal:exact",
  1176. 'definition' => '',
  1177. 'is_obsolete' => 0,
  1178. );
  1179. // TODO: fix the function so it uses prepared statements
  1180. $syntype = tripal_cv_add_cvterm($term, $syncv->name, 0, 1);
  1181. if (!$syntype) {
  1182. watchdog("T_gff3_loader", "Cannot add synonym type: internal:$type", array(), WATCHDOG_WARNING);
  1183. return 0;
  1184. }
  1185. }
  1186. else {
  1187. $syntype = $result[0];
  1188. }
  1189. // iterate through all of the aliases and add each one
  1190. foreach ($aliases as $alias) {
  1191. // check to see if the alias already exists in the synonym table
  1192. // if not, then add it
  1193. $select = array(
  1194. 'name' => $alias,
  1195. 'type_id' => $syntype->cvterm_id,
  1196. );
  1197. $options = array('statement_name' => 'sel_synonym_name_typeid');
  1198. $result = tripal_core_chado_select('synonym', array('*'), $select, $options);
  1199. if (count($result) == 0) {
  1200. $values = array(
  1201. 'name' => $alias,
  1202. 'type_id' => $syntype->cvterm_id,
  1203. 'synonym_sgml' => '',
  1204. );
  1205. $options = array('statement_name' => 'ins_synonym_name_typeid_synonymsgml');
  1206. $success = tripal_core_chado_insert('synonym', $values, $options);
  1207. if (!$success) {
  1208. watchdog("T_gff3_loader", "Cannot add alias $alias to synonym table", array(), WATCHDOG_WARNING);
  1209. return 0;
  1210. }
  1211. $options = array('statement_name' => 'sel_synonym_name_typeid');
  1212. $result = tripal_core_chado_select('synonym', array('*'), $select, $options);
  1213. $synonym = $result[0];
  1214. }
  1215. else {
  1216. $synonym = $result[0];
  1217. }
  1218. // check to see if we have a NULL publication in the pub table. If not,
  1219. // then add one.
  1220. // @coder-ignore: non-drupal schema thus table prefixing does not apply
  1221. $select = array('uniquename' => 'null');
  1222. $options = array('statement_name' => 'sel_pub_uniquename');
  1223. $result = tripal_core_chado_select('pub', array('*'), $select, $options);
  1224. if (count($result) == 0) {
  1225. // prepare the statement
  1226. $psql = "
  1227. PREPARE ins_pub_uniquename_typeid (text, text) AS
  1228. INSERT INTO {pub} (uniquename,type_id) VALUES ('%s',
  1229. (SELECT cvterm_id
  1230. FROM {cvterm} CVT
  1231. INNER JOIN {dbxref} DBX on DBX.dbxref_id = CVT.dbxref_id
  1232. INNER JOIN {db} DB on DB.db_id = DBX.db_id
  1233. WHERE CVT.name = $1 and DB.name = $2)
  1234. ";
  1235. $status = chado_query($psql);
  1236. if (!$status) {
  1237. watchdog("T_gff3_loader", "Cannot prepare statement 'ins_pub_uniquename_typeid", array(), WATCHDOG_WARNING);
  1238. return 0;
  1239. }
  1240. // insert the null pub
  1241. $result = chado_query("EXECUTE ins_pub_uniquename_typeid (:uname, :type_id)",
  1242. array(':uname' => 'null', ':type_id' => 'null'))->fetchObject();
  1243. if (!$result) {
  1244. watchdog("T_gff3_loader", "Cannot add null publication needed for setup of alias", array(), WATCHDOG_WARNING);
  1245. return 0;
  1246. }
  1247. $options = array('statement_name' => 'sel_pub_uniquename');
  1248. $result = tripal_core_chado_select('pub', array('*'), $select, $options);
  1249. $pub = $result[0];
  1250. }
  1251. else {
  1252. $pub = $result[0];
  1253. }
  1254. // check to see if the synonym exists in the feature_synonym table
  1255. // if not, then add it.
  1256. $values = array(
  1257. 'synonym_id' => $synonym->synonym_id,
  1258. 'feature_id' => $feature->feature_id,
  1259. 'pub_id' => $pub->pub_id,
  1260. );
  1261. $columns = array('feature_synonym_id');
  1262. $options = array('statement_name' => 'sel_featuresynonym_syfepu');
  1263. $result = tripal_core_chado_select('feature_synonym', $columns, $values, $options);
  1264. if (count($result) == 0) {
  1265. $values = array(
  1266. 'synonym_id' => $synonym->synonym_id,
  1267. 'feature_id' => $feature->feature_id,
  1268. 'pub_id' => $pub->pub_id,
  1269. );
  1270. $ins_options = array('statement_name' => 'ins_featuresynonym_syfepu');
  1271. $success = tripal_core_chado_insert('feature_synonym', $values, $ins_options);
  1272. if (!$success) {
  1273. watchdog("T_gff3_loader", "Cannot add alias $alias to feature synonym table", array(), WATCHDOG_WARNING);
  1274. return 0;
  1275. }
  1276. }
  1277. }
  1278. return 1;
  1279. }
  1280. /**
  1281. *
  1282. *
  1283. * @ingroup gff3_loader
  1284. */
  1285. function tripal_feature_load_gff3_feature($organism, $analysis_id, $cvterm, $uniquename,
  1286. $name, $residues, $is_analysis = 'f', $is_obsolete = 'f', $add_only, $score) {
  1287. // check to see if the feature already exists
  1288. $feature = NULL;
  1289. $fselect = array(
  1290. 'organism_id' => $organism->organism_id,
  1291. 'uniquename' => $uniquename,
  1292. 'type_id' => $cvterm->cvterm_id
  1293. );
  1294. $options = array('statement_name' => 'sel_feature_orunty');
  1295. $columns = array('feature_id', 'name', 'uniquename', 'seqlen', 'organism_id', 'type_id');
  1296. $result = tripal_core_chado_select('feature', $columns, $fselect, $options);
  1297. if (count($result) > 0) {
  1298. $feature = $result[0];
  1299. }
  1300. if (strcmp($is_obsolete, 'f')==0 or $is_obsolete == 0) {
  1301. $is_obsolete = 'FALSE';
  1302. }
  1303. if (strcmp($is_obsolete, 't')==0 or $is_obsolete == 1) {
  1304. $is_obsolete = 'TRUE';
  1305. }
  1306. if (strcmp($is_analysis, 'f')==0 or $is_analysis == 0) {
  1307. $is_analysis = 'FALSE';
  1308. }
  1309. if (strcmp($is_analysis, 't')==0 or $is_analysis == 1) {
  1310. $is_analysis = 'TRUE';
  1311. }
  1312. // insert the feature if it does not exist otherwise perform an update
  1313. if (!$feature) {
  1314. $values = array(
  1315. 'organism_id' => $organism->organism_id,
  1316. 'name' => $name,
  1317. 'uniquename' => $uniquename,
  1318. // 'residues' => $residues,
  1319. // 'seqlen' => drupal_strlen($residues),
  1320. 'md5checksum' => md5($residues),
  1321. 'type_id' => $cvterm->cvterm_id,
  1322. 'is_analysis' => $is_analysis,
  1323. 'is_obsolete' => $is_obsolete,
  1324. );
  1325. $options = array('statement_name' => 'ins_feature_all');
  1326. $result = tripal_core_chado_insert('feature', $values, $options);
  1327. if (!$result) {
  1328. watchdog("T_gff3_loader", "Failed to insert feature '$uniquename' ($cvterm->name)", array(), WATCHDOG_WARNING);
  1329. return 0;
  1330. }
  1331. }
  1332. elseif (!$add_only) {
  1333. $values = array(
  1334. 'name' => $name,
  1335. // 'residues' => $residues,
  1336. // 'seqlen' => drupal_strlen($residues),
  1337. 'md5checksum' => md5($residues),
  1338. 'is_analysis' => $is_analysis,
  1339. 'is_obsolete' => $is_obsolete,
  1340. );
  1341. $match = array(
  1342. 'organism_id' => $organism->organism_id,
  1343. 'uniquename' => $uniquename,
  1344. 'type_id' => $cvterm->cvterm_id,
  1345. );
  1346. $options = array('statement_name' => 'upd_feature');
  1347. $result = tripal_core_chado_update('feature', $match, $values, $options);
  1348. if (!$result) {
  1349. watchdog("T_gff3_loader", "Failed to update feature '$uniquename' ($cvterm->name)", array(), WATCHDOG_WARNING);
  1350. return 0;
  1351. }
  1352. }
  1353. else {
  1354. // the feature exists and we don't want to update it so return
  1355. // a value of 0. This will stop all downstream property additions
  1356. return 0;
  1357. }
  1358. // get the newly added feature
  1359. $options = array('statement_name' => 'sel_feature_orunty');
  1360. $columns = array('feature_id', 'name', 'uniquename', 'seqlen', 'organism_id', 'type_id');
  1361. $result = tripal_core_chado_select('feature', $columns, $fselect, $options);
  1362. $feature = $result[0];
  1363. // add the analysisfeature entry to the analysisfeature table if it doesn't already exist
  1364. $af_values = array(
  1365. 'analysis_id' => $analysis_id,
  1366. 'feature_id' => $feature->feature_id
  1367. );
  1368. $options = array('statement_name' => 'sel_analysisfeature_analysisid_featureid');
  1369. $afeature = tripal_core_chado_select('analysisfeature', array('analysisfeature_id'), $af_values, $options);
  1370. if (count($afeature)==0) {
  1371. // if a score is available then set that to be the significance field
  1372. if (strcmp($score, '.') != 0) {
  1373. $af_values['significance'] = $score;
  1374. $options = array('statement_name' => 'ins_analysisfeature_analysisid_featureid_significance');
  1375. }
  1376. else {
  1377. $options = array('statement_name' => 'ins_analysisfeature_analysisid_featureid');
  1378. }
  1379. if (!tripal_core_chado_insert('analysisfeature', $af_values, $options)) {
  1380. watchdog("T_gff3_loader", "Could not add analysisfeature record: $analysis_id, $feature->feature_id", array(), WATCHDOG_WARNING);
  1381. }
  1382. }
  1383. else {
  1384. // if a score is available then set that to be the significance field
  1385. $new_vals = array();
  1386. if (strcmp($score, '.')!=0) {
  1387. $new_vals['significance'] = $score;
  1388. }
  1389. else {
  1390. $new_vals['significance'] = '__NULL__';
  1391. }
  1392. if (!$add_only) {
  1393. $options = array('statement_name' => 'upd_analysisfeature');
  1394. $ret = tripal_core_chado_update('analysisfeature', $af_values, $new_vals, $options);
  1395. if (!$ret) {
  1396. watchdog("T_gff3_loader", "Could not update analysisfeature record: $analysis_id, $feature->feature_id", array(), WATCHDOG_WARNING);
  1397. }
  1398. }
  1399. }
  1400. return $feature;
  1401. }
  1402. /**
  1403. *
  1404. *
  1405. * @ingroup gff3_loader
  1406. */
  1407. function tripal_feature_load_gff3_featureloc($feature, $organism, $landmark, $fmin,
  1408. $fmax, $strand, $phase, $is_fmin_partial, $is_fmax_partial, $residue_info, $locgroup,
  1409. $landmark_type_id = '', $landmark_organism_id = '', $create_landmark = 0,
  1410. $landmark_is_target = 0) {
  1411. $select = array(
  1412. 'organism_id' => $landmark_organism_id ? $landmark_organism_id : $organism->organism_id,
  1413. 'uniquename' => $landmark,
  1414. );
  1415. $options = array('statement_name' => 'sel_feature_orun');
  1416. if ($landmark_type_id) {
  1417. $select['type_id'] = $landmark_type_id;
  1418. $options = array('statement_name' => 'sel_feature_orunty');
  1419. }
  1420. $results = tripal_core_chado_select('feature', array('feature_id'), $select, $options);
  1421. $srcfeature = '';
  1422. if (count($results)==0) {
  1423. // so we couldn't find the landmark using the uniquename. Let's try the 'name'.
  1424. // if we return only a single result then we can proceed. Otherwise give an
  1425. $select = array(
  1426. 'organism_id' => $landmark_organism_id ? $landmark_organism_id : $organism->organism_id,
  1427. 'name' => $landmark,
  1428. );
  1429. $options = array('statement_name' => 'sel_feature_orna');
  1430. if ($landmark_type_id) {
  1431. $select['type_id'] = $landmark_type_id;
  1432. $options = array('statement_name' => 'sel_feature_ornaty');
  1433. }
  1434. $results = tripal_core_chado_select('feature', array('feature_id'), $select, $options);
  1435. if (count($results) == 0) {
  1436. // if the landmark is the target feature in a matched alignment then try one more time to
  1437. // find it by querying any feature with the same uniquename. If we find one then use it.
  1438. if ($landmark_is_target) {
  1439. $select = array('uniquename' => $landmark);
  1440. $options = array('statement_name' => 'sel_feature_un');
  1441. $results = tripal_core_chado_select('feature', array('feature_id'), $select, $options);
  1442. if (count($results) == 1) {
  1443. $srcfeature = $results[0];
  1444. }
  1445. }
  1446. if (!$srcfeature) {
  1447. // we couldn't find the landmark feature, so if the user has requested we create it then do so
  1448. // but only if we have a type id
  1449. if ($create_landmark and $landmark_type_id) {
  1450. $values = array(
  1451. 'organism_id' => $landmark_organism_id ? $landmark_organism_id : $organism->organism_id,
  1452. 'name' => $landmark,
  1453. 'uniquename' => $landmark,
  1454. 'type_id' => $landmark_type_id
  1455. );
  1456. $options = array('statement_name' => 'ins_feature_ornaunty');
  1457. $results = tripal_core_chado_insert('feature', $values, $options);
  1458. if (!$results) {
  1459. watchdog("T_gff3_loader", "Cannot find landmark feature: '%landmark', nor could it be inserted",
  1460. array('%landmark' => $landmark), WATCHDOG_WARNING);
  1461. return 0;
  1462. }
  1463. $srcfeature = new stdClass();
  1464. $srcfeature->feature_id = $results['feature_id'];
  1465. }
  1466. else {
  1467. watchdog("T_gff3_loader", "Cannot find unique landmark feature: '%landmark'.",
  1468. array('%landmark' => $landmark), WATCHDOG_WARNING);
  1469. return 0;
  1470. }
  1471. }
  1472. }
  1473. elseif (count($results) > 1) {
  1474. watchdog("T_gff3_loader", "multiple landmarks exist with the name: '%landmark'. Cannot
  1475. resolve which one to use. Cannot add the feature location record",
  1476. array('%landmark' => $landmark), WATCHDOG_WARNING);
  1477. return 0;
  1478. }
  1479. else {
  1480. $srcfeature = $results[0];
  1481. }
  1482. }
  1483. elseif (count($results) > 1) {
  1484. watchdog("T_gff3_loader", "multiple landmarks exist with the name: '%landmark'. Cannot
  1485. resolve which one to use. Cannot add the feature location record",
  1486. array('%landmark' => $landmark), WATCHDOG_WARNING);
  1487. return 0;
  1488. }
  1489. else {
  1490. $srcfeature = $results[0];
  1491. }
  1492. // TODO: create an attribute that recognizes the residue_info,locgroup,
  1493. // is_fmin_partial and is_fmax_partial, right now these are
  1494. // hardcoded to be false and 0 below.
  1495. // check to see if this featureloc already exists, but also keep track of the
  1496. // last rank value
  1497. $rank = 0;
  1498. $exists = 0;
  1499. $select = array('feature_id' => $feature->feature_id);
  1500. $options = array(
  1501. 'statement_name' => 'sel_featureloc_fe',
  1502. 'order_by' => array(
  1503. 'rank' => 'ASC'
  1504. ),
  1505. );
  1506. $locrecs = tripal_core_chado_select('featureloc', array('*'), $select, $options);
  1507. foreach ($locrecs as $featureloc) {
  1508. // it is possible for the featureloc->srcfeature_id to be NULL. This can happen if the srcfeature
  1509. // is not known (according to chado table field descriptions). If it's null then just skip this entry
  1510. if (!$featureloc->srcfeature_id) {
  1511. continue;
  1512. }
  1513. $select = array('feature_id' => $featureloc->srcfeature_id);
  1514. $options = array('statement_name' => 'sel_feature_fe');
  1515. $columns = array('feature_id', 'name');
  1516. $locsfeature = tripal_core_chado_select('feature', $columns, $select, $options);
  1517. // the source feature name and at least the fmin and fmax must be the same
  1518. // for an update of the featureloc, otherwise we'll insert a new record.
  1519. if (strcmp($locsfeature[0]->name, $landmark)==0 and
  1520. ($featureloc->fmin == $fmin or $featureloc->fmax == $fmax)) {
  1521. $match = array('featureloc_id' => $featureloc->featureloc_id);
  1522. $values = array();
  1523. $exists = 1;
  1524. if ($featureloc->fmin != $fmin) {
  1525. $values['fmin'] = $fmin;
  1526. }
  1527. if ($featureloc->fmax != $fmax) {
  1528. $values['fmax'] = $fmax;
  1529. }
  1530. if ($featureloc->strand != $strand) {
  1531. $values['strand'] = $strand;
  1532. }
  1533. if (count($values) > 0) {
  1534. $options = array('statement_name' => 'upd_featureloc_all');
  1535. tripal_core_chado_update('featureloc', $match, $values, $options);
  1536. }
  1537. }
  1538. $rank = $featureloc->rank + 1;
  1539. }
  1540. if (!$exists) {
  1541. // this feature location is new so add it
  1542. if (strcmp($is_fmin_partial, 'f')==0 or !$is_fmin_partial) {
  1543. $is_fmin_partial = 'FALSE';
  1544. }
  1545. elseif (strcmp($is_fmin_partial, 't')==0 or $is_fmin_partial = 1) {
  1546. $is_fmin_partial = 'TRUE';
  1547. }
  1548. if (strcmp($is_fmax_partial, 'f')==0 or !$is_fmax_partial) {
  1549. $is_fmax_partial = 'FALSE';
  1550. }
  1551. elseif (strcmp($is_fmax_partial, 't')==0 or $is_fmax_partial = 1) {
  1552. $is_fmax_partial = 'TRUE';
  1553. }
  1554. $values = array(
  1555. 'feature_id' => $feature->feature_id,
  1556. 'srcfeature_id' => $srcfeature->feature_id,
  1557. 'fmin' => $fmin,
  1558. 'is_fmin_partial' => $is_fmin_partial,
  1559. 'fmax' => $fmax,
  1560. 'is_fmax_partial' => $is_fmax_partial,
  1561. 'strand' => $strand,
  1562. 'residue_info' => $residue_info,
  1563. 'locgroup' => $locgroup,
  1564. 'rank' => $rank
  1565. );
  1566. $options = array('statement_name' => 'ins_featureloc_all');
  1567. if ($phase) {
  1568. $values['phase'] = $phase;
  1569. $options = array('statement_name' => 'ins_featureloc_allphase');
  1570. }
  1571. $success = tripal_core_chado_insert('featureloc', $values, $options);
  1572. if (!$success) {
  1573. watchdog("T_gff3_loader", "Failed to insert featureloc", array(), WATCHDOG_WARNING);
  1574. exit;
  1575. return 0;
  1576. }
  1577. }
  1578. return 1;
  1579. }
  1580. /**
  1581. *
  1582. *
  1583. * @ingroup gff3_loader
  1584. */
  1585. function tripal_feature_load_gff3_property($feature, $property, $value) {
  1586. // first make sure the cvterm exists. if not, then add it
  1587. $select = array(
  1588. 'name' => $property,
  1589. 'cv_id' => array(
  1590. 'name' => 'feature_property',
  1591. ),
  1592. );
  1593. $options = array('statement_name' => 'sel_cvterm_name_cvid');
  1594. $result = tripal_core_chado_select('cvterm', array('*'), $select, $options);
  1595. // if we don't have a property like this already, then add it otherwise, just return
  1596. if (count($result) == 0) {
  1597. $term = array(
  1598. 'id' => "null:$property",
  1599. 'name' => $property,
  1600. 'namespace' => 'feature_property',
  1601. 'is_obsolete' => 0,
  1602. );
  1603. $cvterm = (object) tripal_cv_add_cvterm($term, 'feature_property', 0, 0);
  1604. if (!$cvterm) {
  1605. watchdog("T_gff3_loader", "Cannot add cvterm, $property", array(), WATCHDOG_WARNING);
  1606. return 0;
  1607. }
  1608. }
  1609. else {
  1610. $cvterm = $result[0];
  1611. }
  1612. // check to see if the property already exists for this feature
  1613. // if it does but the value is unique then increment the rank and add it.
  1614. // if the value is not unique then don't add it.
  1615. $add = 1;
  1616. $rank = 0;
  1617. $select = array(
  1618. 'feature_id' => $feature->feature_id,
  1619. 'type_id' => $cvterm->cvterm_id,
  1620. );
  1621. $options = array(
  1622. 'statement_name' => 'sel_featureprop_featureid_typeid',
  1623. 'order_by' => array(
  1624. 'rank' => 'ASC',
  1625. ),
  1626. );
  1627. $results = tripal_core_chado_select('featureprop', array('*'), $select, $options);
  1628. foreach ($results as $prop) {
  1629. if (strcmp($prop->value, $value)==0) {
  1630. $add = NULL; // don't add it, it already exists
  1631. }
  1632. $rank = $prop->rank + 1;
  1633. }
  1634. // add the property if we pass the check above
  1635. if ($add) {
  1636. $values = array(
  1637. 'feature_id' => $feature->feature_id,
  1638. 'type_id' => $cvterm->cvterm_id,
  1639. 'value' => $value,
  1640. 'rank' => $rank,
  1641. );
  1642. $options = array('statement_name' => 'ins_featureprop_all');
  1643. $result = tripal_core_chado_insert('featureprop', $values, $options);
  1644. if (!$result) {
  1645. watchdog("T_gff3_loader", "cannot add featureprop, $property", array(), WATCHDOG_WARNING);
  1646. }
  1647. }
  1648. }
  1649. /*
  1650. *
  1651. */
  1652. function tripal_feature_load_gff3_fasta($fh, $interval, &$num_read, &$intv_read, &$line_num) {
  1653. print "Loading FASTA sequences\n";
  1654. $residues = '';
  1655. $sql = "
  1656. PREPARE sel_gfftemp_un (text) AS
  1657. SELECT feature_id FROM tripal_gff_temp
  1658. WHERE uniquename = $1
  1659. ";
  1660. $status = tripal_core_chado_prepare('sel_gfftemp_un', $sql, array('text'));
  1661. if (!$status) {
  1662. watchdog('T_gff3_loader', 'Cannot prepare statement \'sel_gfftemp_un\'.',
  1663. array(), WATCHDOG_ERROR);
  1664. return '';
  1665. }
  1666. $id = NULL;
  1667. // iterate through the remaining lines of the file
  1668. while ($line = fgets($fh)) {
  1669. $line_num++;
  1670. $size = drupal_strlen($line);
  1671. $num_read += $size;
  1672. $intv_read += $size;
  1673. $line = trim($line);
  1674. // update the job status every 1% features
  1675. if ($job and $intv_read >= $interval) {
  1676. $intv_read = 0;
  1677. $percent = sprintf("%.2f", ($num_read / $filesize) * 100);
  1678. print "Parsing Line $line_num (" . $percent . "%). Memory: " . number_format(memory_get_usage()) . " bytes.\r";
  1679. tripal_job_set_progress($job, intval(($num_read / $filesize) * 100));
  1680. }
  1681. // if we encounter a definition line then get the name, uniquename,
  1682. // accession and relationship subject from the definition line
  1683. if (preg_match('/^>/', $line)) {
  1684. // if we are beginning a new sequence then save the last one we
  1685. // just finished.
  1686. if ($id) {
  1687. $sql = "EXECUTE sel_gfftemp_un('%s')";
  1688. $result = tripal_core_chado_execute_prepared('sel_gfftemp_un', $sql, array($id));
  1689. if (!$result) {
  1690. watchdog('T_gff3_loader', 'Cannot find feature to assign FASTA sequence: %uname',
  1691. array('%uname' => $id), WATCHDOG_WARNING);
  1692. }
  1693. // if we have a feature then add the residues
  1694. else {
  1695. $feature = $result->fetchObject();
  1696. $values = array('residues' => $residues);
  1697. $match = array('feature_id' => $feature->feature_id);
  1698. $options = array('statement_name' => 'upd_feature_re');
  1699. tripal_core_chado_update('feature', $match, $values, $options);
  1700. }
  1701. }
  1702. // get the feature ID for this ID from the tripal_gff_temp table
  1703. $id = preg_replace('/^>(.*)$/', '\1', $line);
  1704. $residues = '';
  1705. }
  1706. else {
  1707. $residues .= trim($line);
  1708. }
  1709. }
  1710. // add in the last sequence
  1711. $sql = "EXECUTE sel_gfftemp_un('%s')";
  1712. $result = tripal_core_chado_execute_prepared('sel_gfftemp_un', $sql, array($id));
  1713. if (!$result) {
  1714. watchdog('T_gff3_loader', 'Cannot find feature to assign FASTA sequence: %uname',
  1715. array('%uname' => $id), WATCHDOG_WARNING);
  1716. }
  1717. // if we have a feature then add the residues
  1718. else {
  1719. $feature = $result->fetchObject();
  1720. $values = array('residues' => $residues);
  1721. $match = array('feature_id' => $feature->feature_id);
  1722. $options = array('statement_name' => 'upd_feature_re');
  1723. tripal_core_chado_update('feature', $match, $values, $options);
  1724. }
  1725. }
  1726. /*
  1727. *
  1728. */
  1729. function tripal_feature_load_gff3_target($feature, $tags, $target_organism_id, $target_type, $create_target, $attr_locgroup) {
  1730. // format is: "target_id start end [strand]", where strand is optional and may be "+" or "-"
  1731. $matched = preg_match('/^(.*?)\s+(\d+)\s+(\d+)(\s+[\+|\-])*$/', trim($tags['Target'][0]), $matches);
  1732. // the organism and type of the target may also be specified as an attribute. If so, then get that
  1733. // information
  1734. $gff_target_organism = array_key_exists('target_organism', $tags) ? $tags['target_organism'][0] : '';
  1735. $gff_target_type = array_key_exists('target_type', $tags) ? $tags['target_type'][0] : '';
  1736. // if we have matches and the Target is in the correct format then load the alignment
  1737. if ($matched) {
  1738. $target_feature = $matches[1];
  1739. $start = $matches[2];
  1740. $end = $matches[3];
  1741. // if we have an optional strand, convert it to a numeric value.
  1742. if ($matches[4]) {
  1743. if (preg_match('/^\+$/', trim($matches[4]))) {
  1744. $target_strand = 1;
  1745. }
  1746. elseif (preg_match('/^\-$/', trim($matches[4]))) {
  1747. $target_strand = -1;
  1748. }
  1749. else {
  1750. $target_strand = 0;
  1751. }
  1752. }
  1753. else {
  1754. $target_strand = 0;
  1755. }
  1756. $target_fmin = $start - 1;
  1757. $target_fmax = $end;
  1758. if ($end < $start) {
  1759. $target_fmin = $end - 1;
  1760. $target_fmax = $start;
  1761. }
  1762. // default the target organism to be the value passed into the function, but if the GFF
  1763. // file species the target organism then use that instead.
  1764. $t_organism_id = $target_organism_id;
  1765. if ($gff_target_organism) {
  1766. // get the genus and species
  1767. $success = preg_match('/^(.*?):(.*?)$/', $gff_target_organism, $matches);
  1768. if ($success) {
  1769. $values = array(
  1770. 'genus' => $matches[1],
  1771. 'species' => $matches[2],
  1772. );
  1773. $options = array('statement_name' => 'sel_organism_gesp');
  1774. $torganism = tripal_core_chado_select('organism', array('organism_id'), $values, $options);
  1775. if (count($torganism) == 1) {
  1776. $t_organism_id = $torganism[0]->organism_id;
  1777. }
  1778. else {
  1779. watchdog('T_gff3_loader', "Cannot find organism for target %target.",
  1780. array('%target' => $gff_target_organism), WATCHDOG_WARNING);
  1781. $t_organism_id = '';
  1782. }
  1783. }
  1784. else {
  1785. watchdog('T_gff3_loader', "The target_organism attribute is improperly formatted: %target.
  1786. It should be target_organism=genus:species.",
  1787. array('%target' => $gff_target_organism), WATCHDOG_WARNING);
  1788. $t_organism_id = '';
  1789. }
  1790. }
  1791. // default the target type to be the value passed into the function, but if the GFF file
  1792. // species the target type then use that instead
  1793. $t_type_id = '';
  1794. if ($target_type) {
  1795. $values = array(
  1796. 'name' => $target_type,
  1797. 'cv_id' => array(
  1798. 'name' => 'sequence',
  1799. )
  1800. );
  1801. $options = array('statement_name' => 'sel_cvterm_nacv');
  1802. $type = tripal_core_chado_select('cvterm', array('cvterm_id'), $values, $options);
  1803. if (count($type) == 1) {
  1804. $t_type_id = $type[0]->cvterm_id;
  1805. }
  1806. else {
  1807. watchdog('T_gff3_loader', "The target type does not exist in the sequence ontology: %type. ",
  1808. array('%type' => $target_type), WATCHDOG_ERROR);
  1809. exit;
  1810. }
  1811. }
  1812. if ($gff_target_type) {
  1813. $values = array(
  1814. 'name' => $gff_target_type,
  1815. 'cv_id' => array(
  1816. 'name' => 'sequence',
  1817. )
  1818. );
  1819. $options = array('statement_name' => 'sel_cvterm_nacv');
  1820. $type = tripal_core_chado_select('cvterm', array('cvterm_id'), $values, $options);
  1821. if (count($type) == 1) {
  1822. $t_type_id = $type[0]->cvterm_id;
  1823. }
  1824. else {
  1825. watchdog('T_gff3_loader', "The target_type attribute does not exist in the sequence ontology: %type. ",
  1826. array('%type' => $gff_target_type), WATCHDOG_WARNING);
  1827. $t_type_id = '';
  1828. }
  1829. }
  1830. // we want to add a featureloc record that uses the target feature as the srcfeature (landmark)
  1831. // and the landmark as the feature.
  1832. tripal_feature_load_gff3_featureloc($feature, $organism, $target_feature, $target_fmin,
  1833. $target_fmax, $target_strand, $phase, $attr_fmin_partial, $attr_fmax_partial, $attr_residue_info,
  1834. $attr_locgroup, $t_type_id, $t_organism_id, $create_target, TRUE);
  1835. }
  1836. // the target attribute is not correctly formatted
  1837. else {
  1838. watchdog('T_gff3_loader', "Could not add 'Target' alignment as it is improperly formatted: '%target'",
  1839. array('%target' => $tags['Target'][0]), WATCHDOG_ERROR);
  1840. }
  1841. }