GFF3Importer.inc 83 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458145914601461146214631464146514661467146814691470147114721473147414751476147714781479148014811482148314841485148614871488148914901491149214931494149514961497149814991500150115021503150415051506150715081509151015111512151315141515151615171518151915201521152215231524152515261527152815291530153115321533153415351536153715381539154015411542154315441545154615471548154915501551155215531554155515561557155815591560156115621563156415651566156715681569157015711572157315741575157615771578157915801581158215831584158515861587158815891590159115921593159415951596159715981599160016011602160316041605160616071608160916101611161216131614161516161617161816191620162116221623162416251626162716281629163016311632163316341635163616371638163916401641164216431644164516461647164816491650165116521653165416551656165716581659166016611662166316641665166616671668166916701671167216731674167516761677167816791680168116821683168416851686168716881689169016911692169316941695169616971698169917001701170217031704170517061707170817091710171117121713171417151716171717181719172017211722172317241725172617271728172917301731173217331734173517361737173817391740174117421743174417451746174717481749175017511752175317541755175617571758175917601761176217631764176517661767176817691770177117721773177417751776177717781779178017811782178317841785178617871788178917901791179217931794179517961797179817991800180118021803180418051806180718081809181018111812181318141815181618171818181918201821182218231824182518261827182818291830183118321833183418351836183718381839184018411842184318441845184618471848184918501851185218531854185518561857185818591860186118621863186418651866186718681869187018711872187318741875187618771878187918801881188218831884188518861887188818891890189118921893189418951896189718981899190019011902190319041905190619071908190919101911191219131914191519161917191819191920192119221923192419251926192719281929193019311932193319341935193619371938193919401941194219431944194519461947194819491950195119521953195419551956195719581959196019611962196319641965196619671968196919701971197219731974197519761977197819791980198119821983198419851986198719881989199019911992199319941995199619971998199920002001200220032004200520062007200820092010201120122013201420152016201720182019202020212022202320242025202620272028202920302031203220332034203520362037203820392040204120422043204420452046204720482049205020512052205320542055205620572058205920602061206220632064206520662067206820692070207120722073207420752076207720782079208020812082208320842085208620872088208920902091209220932094209520962097209820992100210121022103210421052106210721082109211021112112211321142115211621172118211921202121212221232124212521262127212821292130213121322133213421352136213721382139214021412142214321442145214621472148214921502151215221532154215521562157215821592160216121622163216421652166216721682169217021712172217321742175217621772178217921802181218221832184218521862187218821892190219121922193219421952196219721982199220022012202220322042205
  1. <?php
  2. class GFF3Importer extends TripalImporter {
  3. /**
  4. * The name of this loader. This name will be presented to the site
  5. * user.
  6. */
  7. public static $name = 'Chado GFF3 File Loader';
  8. /**
  9. * The machine name for this loader. This name will be used to construct
  10. * the URL for the loader.
  11. */
  12. public static $machine_name = 'chado_gff3_loader';
  13. /**
  14. * A brief description for this loader. This description will be
  15. * presented to the site user.
  16. */
  17. public static $description = 'Import a GFF3 file into Chado';
  18. /**
  19. * An array containing the extensions of allowed file types.
  20. */
  21. public static $file_types = array('gff', 'gff3');
  22. /**
  23. * Provides information to the user about the file upload. Typically this
  24. * may include a description of the file types allowed.
  25. */
  26. public static $upload_description = 'Please provide the GFF3 file.';
  27. /**
  28. * The title that should appear above the upload button.
  29. */
  30. public static $upload_title = 'GFF3 File';
  31. /**
  32. * Text that should appear on the button at the bottom of the importer
  33. * form.
  34. */
  35. public static $button_text = 'Import GFF3 file';
  36. /**
  37. * @see TripalImporter::form()
  38. */
  39. public function form($form, &$form_state) {
  40. // get the list of organisms
  41. $sql = "SELECT * FROM {organism} ORDER BY genus, species";
  42. $org_rset = chado_query($sql);
  43. $organisms = array();
  44. $organisms[''] = '';
  45. while ($organism = $org_rset->fetchObject()) {
  46. $organisms[$organism->organism_id] = "$organism->genus $organism->species ($organism->common_name)";
  47. }
  48. $form['organism_id'] = array(
  49. '#title' => t('Organism'),
  50. '#type' => t('select'),
  51. '#description' => t("Choose the organism to which these sequences are associated"),
  52. '#required' => TRUE,
  53. '#options' => $organisms,
  54. );
  55. // Advanced Options
  56. $form['advanced'] = array(
  57. '#type' => 'fieldset',
  58. '#title' => t('Additional Options'),
  59. '#collapsible' => TRUE,
  60. '#collapsed' => TRUE,
  61. );
  62. $form['advanced']['line_number']= array(
  63. '#type' => 'textfield',
  64. '#title' => t('Start Line Number'),
  65. '#description' => t('Enter the line number in the GFF file where you would like to begin processing. The
  66. first line is line number 1. This option is useful for examining loading problems with large GFF files.'),
  67. '#size' => 10,
  68. );
  69. $form['advanced']['landmark_type'] = array(
  70. '#title' => t('Landmark Type'),
  71. '#type' => t('textfield'),
  72. '#description' => t("Optional. Use this field to specify a Sequence Ontology type
  73. for the landmark sequences in the GFF fie (e.g. 'chromosome'). If the GFF file
  74. contains a '##sequence-region' line that describes the landmark sequences to
  75. which all others are aligned and a type is provided here then the features
  76. will be created if they do not already exist. If they do exist then this
  77. field is not used."),
  78. );
  79. $form['advanced']['alt_id_attr'] = array(
  80. '#title' => t('ID Attribute'),
  81. '#type' => t('textfield'),
  82. '#description' => t("Optional. Sometimes lines in the GFF file are missing the
  83. required ID attribute that specifies the unique name of the feature, but there
  84. may be another attribute that can uniquely identify the feature. If so,
  85. you may specify the name of the attribute to use for the name."),
  86. );
  87. $form['advanced']['skip_protein'] = array(
  88. '#type' => 'checkbox',
  89. '#title' => t('Skip automatic protein creation'),
  90. '#required' => FALSE,
  91. '#description' => t('The GFF loader will automatically create a protein feature for each transcript in the GFF file if a protein feature is missing in the GFF file. Check this box to disable this functionality. Protein features that are specifically present in the GFF will always be created.'),
  92. '#default_value' => 0
  93. );
  94. $form['advanced']['protein_names'] = array(
  95. '#type' => 'fieldset',
  96. '#title' => t('Protein Names'),
  97. '#collapsible' => TRUE,
  98. '#collapsed' => FALSE,
  99. '#weight' => 5,
  100. );
  101. $form['advanced']['protein_names']['re_help'] = array(
  102. '#type' => 'item',
  103. '#markup' => t('A regular expression is an advanced method for extracting information from a string of text.
  104. If your GFF3 file does not contain polypeptide (or protein) features, but contains CDS features, proteins will be automatically created.
  105. By default the loader will give each protein a name based on the name of the corresponding mRNA followed by the "-protein" suffix.
  106. If you want to customize the name of the created protein, you can use the following regex.')
  107. );
  108. $form['advanced']['protein_names']['re_mrna'] = array(
  109. '#type' => 'textfield',
  110. '#title' => t('Regular expression for the mRNA name'),
  111. '#required' => FALSE,
  112. '#description' => t('Enter the regular expression that will extract portions of
  113. the mRNA unique name. For example, for a
  114. mRNA with a unique name finishing by -RX (e.g. SPECIES0000001-RA),
  115. the regular expression would be, "^(.*?)-R([A-Z]+)$".')
  116. );
  117. $form['advanced']['protein_names']['re_protein'] = array(
  118. '#type' => 'textfield',
  119. '#title' => t('Replacement string for the protein name'),
  120. '#required' => FALSE,
  121. '#description' => t('Enter the replacement string that will be used to create
  122. the protein name based on the mRNA regular expression. For example, for a
  123. mRNA regular expression "^(.*?)-R()[A-Z]+)$", the corresponding protein regular
  124. expression would be "$1-P$2".')
  125. );
  126. $form['advanced']['use_transaction']= array(
  127. '#type' => 'checkbox',
  128. '#title' => t('Use a transaction'),
  129. '#required' => FALSE,
  130. '#description' => t('Use a database transaction when loading the GFF file. If an error occurs
  131. the entire datset loaded prior to the failure will be rolled back and will not be available
  132. in the database. If this option is unchecked and failure occurs all records up to the point
  133. of failure will be present in the database.'),
  134. '#default_value' => 1,
  135. );
  136. $form['advanced']['add_only']= array(
  137. '#type' => 'checkbox',
  138. '#title' => t('Import only new features'),
  139. '#required' => FALSE,
  140. '#description' => t('The job will skip features in the GFF file that already
  141. exist in the database and import only new features.'),
  142. );
  143. $form['advanced']['update']= array(
  144. '#type' => 'checkbox',
  145. '#title' => t('Import all and update'),
  146. '#required' => FALSE,
  147. '#default_value' => 'checked',
  148. '#description' => t('Existing features will be updated and new features will be added. Attributes
  149. for a feature that are not present in the GFF but which are present in the
  150. database will not be altered.'),
  151. '#default_value' => 1,
  152. );
  153. // SPF: there are bugs in refreshing and removing features. The bugs arise
  154. // if a feature in the GFF does not have a uniquename. GenSAS will auto
  155. // generate this uniquename and it will not be the same as a previous
  156. // load because it uses the date. This causes orphaned CDS/exons, UTRs
  157. // to be left behind during a delete or refresh. So, the short term
  158. // fix is to remove these options.
  159. // $form['import_options']['refresh']= array(
  160. // '#type' => 'checkbox',
  161. // '#title' => t('Import all and replace'),
  162. // '#required' => FALSE,
  163. // '#description' => t('Existing features will be updated and feature properties not
  164. // present in the GFF file will be removed.'),
  165. // );
  166. // $form['import_options']['remove']= array(
  167. // '#type' => 'checkbox',
  168. // '#title' => t('Delete features'),
  169. // '#required' => FALSE,
  170. // '#description' => t('Features present in the GFF file that exist in the database
  171. // will be removed rather than imported'),
  172. // );
  173. $form['advanced']['create_organism']= array(
  174. '#type' => 'checkbox',
  175. '#title' => t('Create organism'),
  176. '#required' => FALSE,
  177. '#description' => t('The Tripal GFF loader supports the "organism" attribute. This allows features of a
  178. different organism to be aligned to the landmark sequence of another species. The format of the
  179. attribute is "organism=[genus]:[species]", where [genus] is the organism\'s genus and [species] is the
  180. species name. Check this box to automatically add the organism to the database if it does not already exists.
  181. Otherwise lines with an oraganism attribute where the organism is not present in the database will be skipped.'),
  182. );
  183. $form['advanced']['targets'] = array(
  184. '#type' => 'fieldset',
  185. '#title' => t('Targets'),
  186. '#collapsible' => TRUE,
  187. '#collapsed' => FALSE,
  188. '#weight' => 1,
  189. );
  190. $form['advanced']['targets']['adesc'] = array(
  191. '#markup' => t("When alignments are represented in the GFF file (e.g. such as
  192. alignments of cDNA sequences to a whole genome, or blast matches), they are
  193. represented using two feature types: 'match' (or cDNA_match, EST_match, etc.)
  194. and 'match_part'. These features may also have a 'Target' attribute to
  195. specify the sequence that is being aligned.
  196. However, the organism to which the aligned sequence belongs may not be present in the
  197. GFF file. Here you can specify the organism and feature type of the target sequences.
  198. The options here will apply to all targets unless the organism and type are explicity
  199. set in the GFF file using the 'target_organism' and 'target_type' attributes."),
  200. );
  201. $form['advanced']['targets']['target_organism_id'] = array(
  202. '#title' => t('Target Organism'),
  203. '#type' => t('select'),
  204. '#description' => t("Optional. Choose the organism to which target sequences belong.
  205. Select this only if target sequences belong to a different organism than the
  206. one specified above. And only choose an organism here if all of the target sequences
  207. belong to the same species. If the targets in the GFF file belong to multiple
  208. different species then the organism must be specified using the 'target_organism=genus:species'
  209. attribute in the GFF file."),
  210. '#options' => $organisms,
  211. );
  212. $form['advanced']['targets']['target_type'] = array(
  213. '#title' => t('Target Type'),
  214. '#type' => t('textfield'),
  215. '#description' => t("Optional. If the unique name for a target sequence is not unique (e.g. a protein
  216. and an mRNA have the same name) then you must specify the type for all targets in the GFF file. If
  217. the targets are of different types then the type must be specified using the 'target_type=type' attribute
  218. in the GFF file. This must be a valid Sequence Ontology (SO) term."),
  219. );
  220. $form['advanced']['targets']['create_target']= array(
  221. '#type' => 'checkbox',
  222. '#title' => t('Create Target'),
  223. '#required' => FALSE,
  224. '#description' => t("If the target feature cannot be found, create one using the organism and type specified above, or
  225. using the 'target_organism' and 'target_type' fields specified in the GFF file. Values specified in the
  226. GFF file take precedence over those specified above."),
  227. );
  228. return $form;
  229. }
  230. /**
  231. * @see TripalImporter::formValidate()
  232. */
  233. public function formValidate($form, &$form_state) {
  234. $organism_id = $form_state['values']['organism_id'];
  235. $target_organism_id = $form_state['values']['target_organism_id'];
  236. $target_type = trim($form_state['values']['target_type']);
  237. $create_target = $form_state['values']['create_target'];
  238. $create_organism = $form_state['values']['create_organism'];
  239. $add_only = $form_state['values']['add_only'];
  240. $update = $form_state['values']['update'];
  241. $refresh = 0; //$form_state['values']['refresh'];
  242. $remove = 0; //$form_state['values']['remove'];
  243. $use_transaction = $form_state['values']['use_transaction'];
  244. $line_number = trim($form_state['values']['line_number']);
  245. $landmark_type = trim($form_state['values']['landmark_type']);
  246. $alt_id_attr = trim($form_state['values']['alt_id_attr']);
  247. $re_mrna = trim($form_state['values']['re_mrna']);
  248. $re_protein = trim($form_state['values']['re_protein']);
  249. // @coder-ignore: there are no functions being called here
  250. if (($add_only AND ($update OR $refresh OR $remove)) OR
  251. ($update AND ($add_only OR $refresh OR $remove)) OR
  252. ($refresh AND ($update OR $add_only OR $remove)) OR
  253. ($remove AND ($update OR $refresh OR $add_only))) {
  254. form_set_error('add_only', t("Please select only one checkbox from the import options section"));
  255. }
  256. if ($line_number and !is_numeric($line_number) or $line_number < 0) {
  257. form_set_error('line_number', t("Please provide an integer line number greater than zero."));
  258. }
  259. if (!($re_mrna and $re_protein) and ($re_mrna or $re_protein)) {
  260. form_set_error('re_uname', t("You must provide both a regular expression for mRNA and a replacement string for protein"));
  261. }
  262. // check the regular expression to make sure it is valid
  263. set_error_handler(function() {}, E_WARNING);
  264. $result_re = preg_match("/" . $re_mrna . "/", null);
  265. $result = preg_replace("/" . $re_mrna . "/", $re_protein, null);
  266. restore_error_handler();
  267. if ($result_re === FALSE) {
  268. form_set_error('re_mrna', 'Invalid regular expression.');
  269. } else if ($result === FALSE) {
  270. form_set_error('re_protein', 'Invalid replacement string.');
  271. }
  272. }
  273. /**
  274. * @see TripalImporter::run()
  275. */
  276. public function run() {
  277. $arguments = $this->arguments['run_args'];
  278. $file_path = $this->arguments['files'][0]['file_path'];
  279. $organism_id = $arguments['organism_id'];
  280. $analysis_id = $arguments['analysis_id'];
  281. $add_only = $arguments['add_only'];
  282. $update = $arguments['update'];
  283. $refresh = FALSE;
  284. $remove = FALSE;
  285. $use_transaction = $arguments['use_transaction'];
  286. $target_organism_id = $arguments['target_organism_id'];
  287. $target_type = $arguments['target_type'];
  288. $create_target = $arguments['create_target'];
  289. $start_line = $arguments['line_number'];
  290. $landmark_type = $arguments['landmark_type'];
  291. $alt_id_attr = $arguments['alt_id_attr'];
  292. $create_organism = $arguments['create_organism'];
  293. $re_mrna = $arguments['re_mrna'];
  294. $re_protein = $arguments['re_protein'];
  295. $skip_protein = $arguments['skip_protein'];
  296. $this->loadGFF3($file_path, $organism_id, $analysis_id,
  297. $add_only, $update, $refresh, $remove, $use_transaction,
  298. $target_organism_id, $target_type, $create_target,
  299. $start_line, $landmark_type, $alt_id_attr, $create_organism,
  300. $re_mrna, $re_protein, $skip_protein);
  301. }
  302. /**
  303. * Actually load a GFF3 file. This is the function called by tripal jobs
  304. *
  305. * @param $gff_file
  306. * The full path to the GFF file on the filesystem
  307. * @param $organism_id
  308. * The organism_id of the organism to which the features in the GFF belong
  309. * @param $analysis_id
  310. * The anlaysis_id of the analysis from which the features in the GFF were generated
  311. * @param $add_only
  312. * Set to 1 if feature should be added only. In the case where a feature
  313. * already exists, it will not be updated. Default is 0
  314. * @param $update
  315. * Set to 1 to update existing features. New features will be added. Attributes
  316. * for a feature that are not present in the GFF but which are present in the
  317. * database will not be altered. Default is 1
  318. * @param $refresh
  319. * Set to 1 to update existing features. New features will be added. Attributes
  320. * for a feature that are not present in the GFF but which are present in the
  321. * database will be removed. Default is 0
  322. * @param $remove
  323. * Set to 1 to remove features present in the GFF file that exist in the database.
  324. * Default is 0.
  325. * @param $use_transaction
  326. * Set to 1 to use a transaction when loading the GFF. Any failure during
  327. * loading will result in the rollback of any changes. Default is 1.
  328. * @param $target_organism_id
  329. * If the GFF file contains a 'Target' attribute then the feature and the
  330. * target will have an alignment created, but to find the proper target
  331. * feature the target organism must also be known. If different from the
  332. * organism specified for the GFF file, then use this argument to specify
  333. * the target organism. Only use this argument if all target sequences belong
  334. * to the same species. If the targets in the GFF file belong to multiple
  335. * different species then the organism must be specified using the
  336. * 'target_organism=genus:species' attribute in the GFF file. Default is NULL.
  337. * @param $target_type
  338. * If the GFF file contains a 'Target' attribute then the feature and the
  339. * target will have an alignment created, but to find the proper target
  340. * feature the target organism must also be known. This can be used to
  341. * specify the target feature type to help with identification of the target
  342. * feature. Only use this argument if all target sequences types are the same.
  343. * If the targets are of different types then the type must be specified using
  344. * the 'target_type=type' attribute in the GFF file. This must be a valid
  345. * Sequence Ontology (SO) term. Default is NULL
  346. * @param $create_target
  347. * Set to 1 to create the target feature if it cannot be found in the
  348. * database. Default is 0
  349. * @param $start_line
  350. * Set this to the line in the GFF file where importing should start. This
  351. * is useful for testing and debugging GFF files that may have problems and
  352. * you want to start at a particular line to speed testing. Default = 1
  353. * @param $landmark_type
  354. * Use this argument to specify a Sequence Ontology term name for the landmark
  355. * sequences in the GFF fie (e.g. 'chromosome'), if the GFF file contains a
  356. * '##sequence-region' line that describes the landmark sequences. Default = ''
  357. * @param $alt_id_attr
  358. * Sometimes lines in the GFF file are missing the required ID attribute that
  359. * specifies the unique name of the feature. If so, you may specify the
  360. * name of an existing attribute to use for the ID.
  361. * @param $create_organism
  362. * The Tripal GFF loader supports the "organism" attribute. This allows
  363. * features of a different organism to be aligned to the landmark sequence of
  364. * another species. The format of the attribute is "organism=[genus]:[species]",
  365. * where [genus] is the organism's genus and [species] is the species name.
  366. * Check this box to automatically add the organism to the database if it does
  367. * not already exists. Otherwise lines with an oraganism attribute where the
  368. * organism is not present in the database will be skipped.
  369. * @param $re_mrna A
  370. * regular expression to extract portions from mRNA id
  371. * @param $re_protein A
  372. * replacement string to generate the protein id
  373. * @PARAM $skip_protein
  374. * BOOL: Will the loader create the proteins inferred from the CDS?
  375. *
  376. * @ingroup gff3_loader
  377. */
  378. private function loadGFF3($gff_file, $organism_id, $analysis_id,
  379. $add_only = 0, $update = 1, $refresh = 0, $remove = 0, $use_transaction = 1,
  380. $target_organism_id = NULL, $target_type = NULL, $create_target = 0,
  381. $start_line = 1, $landmark_type = '', $alt_id_attr = '', $create_organism = FALSE,
  382. $re_mrna = '', $re_protein = '', $skip_protein = 0) {
  383. $ret = array();
  384. $date = getdate();
  385. // An array that stores CVterms that have been looked up so we don't have
  386. // to do the database query every time.
  387. $cvterm_lookup = array();
  388. // An array that stores Landmarks that have been looked up so we don't have
  389. // to do the database query every time.
  390. $landmark_lookup = array();
  391. // empty the temp tables
  392. $sql = "DELETE FROM {tripal_gff_temp}";
  393. chado_query($sql);
  394. $sql = "DELETE FROM {tripal_gffcds_temp}";
  395. chado_query($sql);
  396. $sql = "DELETE FROM {tripal_gffprotein_temp}";
  397. chado_query($sql);
  398. // check to see if the file is located local to Drupal
  399. $dfile = $_SERVER['DOCUMENT_ROOT'] . base_path() . $gff_file;
  400. if (!file_exists($dfile)) {
  401. // if not local to Drupal, the file must be someplace else, just use
  402. // the full path provided
  403. $dfile = $gff_file;
  404. }
  405. if (!file_exists($dfile)) {
  406. throw new Exception(t("Cannot find the file: !dfile", array('!dfile' => $dfile)));
  407. }
  408. $this->logMessage("Opening !gff_file", array('!gff_file' => $gff_file));
  409. //$lines = file($dfile,FILE_SKIP_EMPTY_LINES);
  410. $fh = fopen($dfile, 'r');
  411. if (!$fh) {
  412. throw new Exception(t("Cannot open file: !dfile", array('!dfile' => $dfile)));
  413. }
  414. $filesize = filesize($dfile);
  415. $this->setTotalItems($filesize);
  416. // get the controlled vocaubulary that we'll be using. The
  417. // default is the 'sequence' ontology
  418. $sql = "SELECT * FROM {cv} WHERE name = :cvname";
  419. $cv = chado_query($sql, array(':cvname' => 'sequence'))->fetchObject();
  420. if (!$cv) {
  421. throw new Exception(t("Cannot find the 'sequence' ontology", array()));
  422. }
  423. // get the organism for which this GFF3 file belongs
  424. $sql = "SELECT * FROM {organism} WHERE organism_id = :organism_id";
  425. $organism = chado_query($sql, array(':organism_id' => $organism_id))->fetchObject();
  426. $in_fasta = 0;
  427. $line_num = 0;
  428. $num_read = 0;
  429. // prepare the statement used to get the cvterm for each feature.
  430. $sel_cvterm_sql = "
  431. SELECT CVT.cvterm_id, CVT.cv_id, CVT.name, CVT.definition,
  432. CVT.dbxref_id, CVT.is_obsolete, CVT.is_relationshiptype
  433. FROM {cvterm} CVT
  434. INNER JOIN {cv} CV on CVT.cv_id = CV.cv_id
  435. LEFT JOIN {cvtermsynonym} CVTS on CVTS.cvterm_id = CVT.cvterm_id
  436. WHERE CV.cv_id = :cv_id and
  437. (lower(CVT.name) = lower(:name) or lower(CVTS.synonym) = lower(:synonym))
  438. ";
  439. // If a landmark type was provided then pre-retrieve that.
  440. if ($landmark_type) {
  441. $query = array(
  442. ':cv_id' => $cv->cv_id,
  443. ':name' => $landmark_type,
  444. ':synonym' => $landmark_type
  445. );
  446. $result = chado_query($sel_cvterm_sql, $query);
  447. $landmark_cvterm = $result->fetchObject();
  448. if (!$landmark_cvterm) {
  449. throw new Exception(t('Cannot find landmark feature type \'%landmark_type\'.', array('%landmark_type' => $landmark_type)));
  450. }
  451. }
  452. // iterate through each line of the GFF file
  453. while ($line = fgets($fh)) {
  454. $line_num++;
  455. $size = drupal_strlen($line);
  456. $this->addItemsHandled($size);
  457. $num_read += $size;
  458. if ($line_num < $start_line) {
  459. continue;
  460. }
  461. // check to see if we have FASTA section, if so then set the variable
  462. // to start parsing
  463. if (preg_match('/^##FASTA/i', $line)) {
  464. $this->logMessage("Parsing FASTA portion...");
  465. if ($remove) {
  466. // we're done because this is a delete operation so break out of the loop.
  467. break;
  468. }
  469. $this->loadFasta($fh, $interval, $num_read, $line_num, $filesize);
  470. continue;
  471. }
  472. // if the ##sequence-region line is present then we want to add a new feature
  473. if (preg_match('/^##sequence-region (.*?) (\d+) (\d+)$/i', $line, $region_matches)) {
  474. $rid = $region_matches[1];
  475. $rstart = $region_matches[2];
  476. $rend = $region_matches[3];
  477. if ($landmark_type) {
  478. $this->loadFeature($organism, $analysis_id, $landmark_cvterm, $rid,
  479. $rid, '', 'f', 'f', 1, 0);
  480. }
  481. continue;
  482. }
  483. // skip comments
  484. if (preg_match('/^#/', $line)) {
  485. continue;
  486. }
  487. // skip empty lines
  488. if (preg_match('/^\s*$/', $line)) {
  489. continue;
  490. }
  491. // get the columns
  492. $cols = explode("\t", $line);
  493. if (sizeof($cols) != 9) {
  494. throw new Exception(t('Improper number of columns on line %line_num', array('%line_num' => $line_num)));
  495. }
  496. // get the column values
  497. $landmark = $cols[0];
  498. $source = $cols[1];
  499. $type = $cols[2];
  500. $start = $cols[3];
  501. $end = $cols[4];
  502. $score = $cols[5];
  503. $strand = $cols[6];
  504. $phase = $cols[7];
  505. $attrs = explode(";", $cols[8]); // split by a semicolon
  506. // ready the start and stop for chado. Chado expects these positions
  507. // to be zero-based, so we substract 1 from the fmin
  508. $fmin = $start - 1;
  509. $fmax = $end;
  510. if ($end < $start) {
  511. $fmin = $end - 1;
  512. $fmax = $start;
  513. }
  514. // format the strand for chado
  515. if (strcmp($strand, '.') == 0) {
  516. $strand = 0;
  517. }
  518. elseif (strcmp($strand, '+') == 0) {
  519. $strand = 1;
  520. }
  521. elseif (strcmp($strand, '-') == 0) {
  522. $strand = -1;
  523. }
  524. if (strcmp($phase, '.') == 0) {
  525. if ($type == 'CDS') {
  526. $phase = '0';
  527. }
  528. else {
  529. $phase = '';
  530. }
  531. }
  532. if (array_key_exists($type, $cvterm_lookup)) {
  533. $cvterm = $cvterm_lookup[$type];
  534. }
  535. else {
  536. $result = chado_query($sel_cvterm_sql, array(':cv_id' => $cv->cv_id, ':name' => $type, ':synonym' => $type));
  537. $cvterm = $result->fetchObject();
  538. $cvterm_lookup[$type] = $cvterm;
  539. if (!$cvterm) {
  540. throw new Exception(t('Cannot find feature term \'%type\' on line %line_num of the GFF file',
  541. array('%type' => $type, '%line_num' => $line_num)));
  542. }
  543. }
  544. // break apart each of the attributes
  545. $tags = array();
  546. $attr_name = '';
  547. $attr_uniquename = '';
  548. $attr_residue_info = '';
  549. $attr_locgroup = 0;
  550. $attr_fmin_partial = 'f';
  551. $attr_fmax_partial = 'f';
  552. $attr_is_obsolete = 'f';
  553. $attr_is_analysis = 'f';
  554. $attr_others = [];
  555. $residues = '';
  556. // the organism to which a feature belongs can be set in the GFF
  557. // file using the 'organism' attribute. By default we
  558. // set the $feature_organism variable to the default organism for the landmark
  559. $attr_organism = '';
  560. $feature_organism = $organism;
  561. foreach ($attrs as $attr) {
  562. $attr = rtrim($attr);
  563. $attr = ltrim($attr);
  564. if (strcmp($attr, '')==0) {
  565. continue;
  566. }
  567. if (!preg_match('/^[^\=]+\=.+$/', $attr)) {
  568. throw new Exception(t('Attribute is not correctly formatted on line %line_num: %attr',
  569. array('%line_num' => $line_num, '%attr' => $attr)));
  570. }
  571. // break apart each tag
  572. $tag = preg_split("/=/", $attr, 2); // split by equals sign
  573. // multiple instances of an attribute are separated by commas
  574. $tag_name = $tag[0];
  575. if (!array_key_exists($tag_name, $tags)) {
  576. $tags[$tag_name] = array();
  577. }
  578. $tags[$tag_name] = array_merge($tags[$tag_name], explode(",", $tag[1])); // split by comma
  579. // replace the URL escape codes for each tag
  580. for ($i = 0; $i < count($tags[$tag_name]); $i++) {
  581. $tags[$tag_name][$i] = urldecode($tags[$tag_name][$i]);
  582. }
  583. // get the name and ID tags
  584. $skip_feature = 0; // if there is a problem with any of the attributes this variable gets set
  585. if (strcmp($tag_name, 'ID') == 0) {
  586. $attr_uniquename = urldecode($tag[1]);
  587. }
  588. elseif (strcmp($tag_name, 'Name') == 0) {
  589. $attr_name = urldecode($tag[1]);
  590. }
  591. elseif (strcmp($tag_name, 'organism') == 0) {
  592. $attr_organism = urldecode($tag[1]);
  593. $org_matches = array();
  594. if (preg_match('/^(.*?):(.*?)$/', $attr_organism, $org_matches)) {
  595. $values = array(
  596. 'genus' => $org_matches[1],
  597. 'species' => $org_matches[2],
  598. );
  599. $org = chado_select_record('organism', array("*"), $values);
  600. if (count($org) == 0) {
  601. if ($create_organism) {
  602. $feature_organism = (object) chado_insert_record('organism', $values);
  603. if (!$feature_organism) {
  604. $this->logMessage("Could not add the organism, '%org', from line %line. Skipping this line.",
  605. array('%org' => $attr_organism, '%line' => $line_num), TRIPAL_ERROR);
  606. $skip_feature = 1;
  607. }
  608. }
  609. else {
  610. $this->logMessage("The organism attribute '%org' on line %line does not exist. Skipping this line.",
  611. array('%org' => $attr_organism, '%line' => $line_num), TRIPAL_ERROR);
  612. $skip_feature = 1;
  613. }
  614. }
  615. else {
  616. // We found the organism in the database so use it.
  617. $feature_organism = $org[0];
  618. }
  619. }
  620. else {
  621. $this->logMessage("The organism attribute '%org' on line %line is not properly formated. It " .
  622. "should be of the form: organism=Genus:species. Skipping this line.",
  623. array('%org' => $attr_organism, '%line' => $line_num), TRIPAL_ERROR);
  624. $skip_feature = 1;
  625. }
  626. }
  627. // Get the list of non-reserved attributes.
  628. elseif (strcmp($tag_name, 'Alias') != 0 and strcmp($tag_name, 'Parent') != 0 and
  629. strcmp($tag_name, 'Target') != 0 and strcmp($tag_name, 'Gap') != 0 and
  630. strcmp($tag_name, 'Derives_from') != 0 and strcmp($tag_name, 'Note') != 0 and
  631. strcmp($tag_name, 'Dbxref') != 0 and strcmp($tag_name, 'Ontology_term') != 0 and
  632. strcmp($tag_name, 'Is_circular') != 0 and strcmp($tag_name, 'target_organism') != 0 and
  633. strcmp($tag_name, 'target_type') != 0 and strcmp($tag_name, 'organism' != 0)) {
  634. foreach ($tags[$tag_name] as $value) {
  635. $attr_others[$tag_name][] = $value;
  636. }
  637. }
  638. }
  639. // If neither name nor uniquename are provided then generate one.
  640. if (!$attr_uniquename and !$attr_name) {
  641. // Check if an alternate ID field is suggested, if so, then use
  642. // that for the name.
  643. if (array_key_exists($alt_id_attr, $tags)) {
  644. $attr_uniquename = $tags[$alt_id_attr][0];
  645. $attr_name = $attr_uniquename;
  646. }
  647. // If the row has a parent then generate a uniquename using the parent name
  648. // add the date to the name in the event there are more than one child with
  649. // the same parent.
  650. elseif (array_key_exists('Parent', $tags)) {
  651. $attr_uniquename = $tags['Parent'][0] . "-$type-$landmark-" . $date[0] . ":" . ($fmin + 1) . ".." . $fmax;
  652. $attr_name = $attr_uniquename;
  653. }
  654. // Generate a unique name based on the date, type and location
  655. // and set the name to simply be the type.
  656. else {
  657. $attr_uniquename = $date[0] . "-$type-$landmark:" . ($fmin + 1) . ".." . $fmax;
  658. $attr_name = $type;
  659. }
  660. }
  661. // If a name is not specified then use the unique name as the name
  662. if (strcmp($attr_name, '') == 0) {
  663. $attr_name = $attr_uniquename;
  664. }
  665. // If an ID attribute is not specified then we must generate a
  666. // unique ID. Do this by combining the attribute name with the date
  667. // and line number.
  668. if (!$attr_uniquename) {
  669. $attr_uniquename = $attr_name . '-' . $date[0] . '-' . $line_num;
  670. }
  671. // Make sure the landmark sequence exists in the database. If the user
  672. // has not specified a landmark type (and it's not required in the GFF
  673. // format) then we don't know the type of the landmark so we'll hope
  674. // that it's unique across all types for the organism. Only do this
  675. // test if the landmark and the feature are different.
  676. if (!$remove and !(strcmp($landmark, $attr_uniquename) == 0 or strcmp($landmark, $attr_name) == 0) and !in_array($landmark, $landmark_lookup)) {
  677. $select = array(
  678. 'organism_id' => $organism->organism_id,
  679. 'uniquename' => $landmark,
  680. );
  681. $columns = array('count(*) as num_landmarks');
  682. if ($landmark_type) {
  683. $select['type_id'] = array(
  684. 'name' => $landmark_type,
  685. );
  686. }
  687. $count = chado_select_record('feature', $columns, $select);
  688. if (!$count or count($count) == 0 or $count[0]->num_landmarks == 0) {
  689. // now look for the landmark using the name rather than uniquename.
  690. $select = array(
  691. 'organism_id' => $organism->organism_id,
  692. 'name' => $landmark,
  693. );
  694. $columns = array('count(*) as num_landmarks');
  695. if ($landmark_type) {
  696. $select['type_id'] = array(
  697. 'name' => $landmark_type,
  698. );
  699. }
  700. $count = chado_select_record('feature', $columns, $select);
  701. if (!$count or count($count) == 0 or $count[0]->num_landmarks == 0) {
  702. throw new Exception(t("The landmark '%landmark' cannot be found for this organism (%species) " .
  703. "Please add the landmark and then retry the import of this GFF3 " .
  704. "file", array('%landmark' => $landmark, '%species' => $organism->genus . " " . $organism->species)));
  705. }
  706. elseif ($count[0]->num_landmarks > 1) {
  707. throw new Exception(t("The landmark '%landmark' has more than one entry for this organism (%species) " .
  708. "Cannot continue", array('%landmark' => $landmark, '%species' => $organism->genus . " " . $organism->species)));
  709. }
  710. }
  711. if ($count[0]->num_landmarks > 1) {
  712. throw new Exception(t("The landmark '%landmark' is not unique for this organism. " .
  713. "The features cannot be associated", array('%landmark' => $landmark)));
  714. }
  715. // The landmark was found, remember it
  716. $landmark_lookup[] = $landmark;
  717. }
  718. /*
  719. // If the option is to remove or refresh then we want to remove
  720. // the feature from the database.
  721. if ($remove or $refresh) {
  722. // Next remove the feature itself.
  723. $sql = "DELETE FROM {feature}
  724. WHERE organism_id = %d and uniquename = '%s' and type_id = %d";
  725. $match = array(
  726. 'organism_id' => $feature_organism->organism_id,
  727. 'uniquename' => $attr_uniquename,
  728. 'type_id' => $cvterm->cvterm_id
  729. );
  730. $result = chado_delete_record('feature', $match);
  731. if (!$result) {
  732. $this->logMessage("Cannot delete feature %attr_uniquename",
  733. array('%attr_uniquename' => $attr_uniquename), TRIPAL_ERROR);
  734. }
  735. $feature = 0;
  736. unset($result);
  737. }
  738. */
  739. // Add or update the feature and all properties.
  740. if ($update or $refresh or $add_only) {
  741. // Add/update the feature.
  742. $feature = $this->loadFeature($feature_organism, $analysis_id, $cvterm,
  743. $attr_uniquename, $attr_name, $residues, $attr_is_analysis,
  744. $attr_is_obsolete, $add_only, $score);
  745. if ($feature) {
  746. // Add a record for this feature to the tripal_gff_temp table for
  747. // later lookup.
  748. $values = array(
  749. 'feature_id' => $feature->feature_id,
  750. 'organism_id' => $feature->organism_id,
  751. 'type_name' => $type,
  752. 'uniquename' => $feature->uniquename
  753. );
  754. // make sure this record doesn't already exist in our temp table
  755. $results = chado_select_record('tripal_gff_temp', array('*'), $values);
  756. if (count($results) == 0) {
  757. $result = chado_insert_record('tripal_gff_temp', $values);
  758. if (!$result) {
  759. throw new Exception(t("Cound not save record in temporary table, Cannot continue.", array()));
  760. }
  761. }
  762. // add/update the featureloc if the landmark and the ID are not the same
  763. // if they are the same then this entry in the GFF is probably a landmark identifier
  764. if (strcmp($landmark, $attr_uniquename) !=0 ) {
  765. $this->loadFeatureLoc($feature, $organism,
  766. $landmark, $fmin, $fmax, $strand, $phase, $attr_fmin_partial,
  767. $attr_fmax_partial, $attr_residue_info, $attr_locgroup);
  768. }
  769. // add any aliases for this feature
  770. if (array_key_exists('Alias', $tags)) {
  771. $this->loadAlias($feature, $tags['Alias']);
  772. }
  773. // add any dbxrefs for this feature
  774. if (array_key_exists('Dbxref', $tags)) {
  775. $this->loadDbxref($feature, $tags['Dbxref']);
  776. }
  777. // add any ontology terms for this feature
  778. if (array_key_exists('Ontology_term', $tags)) {
  779. $this->loadOntology($feature, $tags['Ontology_term']);
  780. }
  781. // add parent relationships
  782. if (array_key_exists('Parent', $tags)) {
  783. $this->loadParents($feature, $cvterm, $tags['Parent'],
  784. $feature_organism->organism_id, $strand, $phase, $fmin, $fmax);
  785. }
  786. // add target relationships
  787. if (array_key_exists('Target', $tags)) {
  788. $this->loadTarget($feature, $tags, $target_organism_id, $target_type, $create_target, $attr_locgroup);
  789. }
  790. // add gap information. This goes in simply as a property
  791. if (array_key_exists('Gap', $tags)) {
  792. foreach ($tags['Gap'] as $value) {
  793. $this->loadProperty($feature, 'Gap', $value);
  794. }
  795. }
  796. // add notes. This goes in simply as a property
  797. if (array_key_exists('Note', $tags)) {
  798. foreach ($tags['Note'] as $value) {
  799. $this->loadProperty($feature, 'Note', $value);
  800. }
  801. }
  802. // add the Derives_from relationship (e.g. polycistronic genes).
  803. if (array_key_exists('Derives_from', $tags)) {
  804. $this->loadDerivesFrom($feature, $cvterm, $tags['Derives_from'][0],
  805. $feature_organism, $fmin, $fmax);
  806. }
  807. // add in the GFF3_source dbxref so that GBrowse can find the feature using the source column
  808. $source_ref = array('GFF_source:' . $source);
  809. $this->loadDbxref($feature, $source_ref);
  810. // add any additional attributes
  811. if ($attr_others) {
  812. foreach ($attr_others as $tag_name => $values) {
  813. foreach ($values as $value) {
  814. $this->loadProperty($feature, $tag_name, $value);
  815. }
  816. }
  817. }
  818. }
  819. }
  820. }
  821. // Do some last bit of processing.
  822. if (!$remove) {
  823. // First, add any protein sequences if needed.
  824. $sql = "SELECT feature_id FROM {tripal_gffcds_temp} LIMIT 1 OFFSET 1";
  825. $has_cds = chado_query($sql)->fetchField();
  826. if ($has_cds) {
  827. $this->logMessage("\nAdding protein sequences if CDS exist and no proteins in GFF...");
  828. $sql = "
  829. SELECT F.feature_id, F.name, F.uniquename, TGCT.strand,
  830. CVT.cvterm_id, CVT.name as feature_type,
  831. min(TGCT.fmin) as fmin, max(TGCT.fmax) as fmax,
  832. TGPT.feature_id as protein_id, TGPT.fmin as protein_fmin,
  833. TGPT.fmax as protein_fmax, FLM.uniquename as landmark
  834. FROM {tripal_gffcds_temp} TGCT
  835. INNER JOIN {feature} F on F.feature_id = TGCT.parent_id
  836. INNER JOIN {cvterm} CVT on CVT.cvterm_id = F.type_id
  837. INNER JOIN {featureloc} L on F.feature_id = L.feature_id
  838. INNER JOIN {feature} FLM on L.srcfeature_id = FLM.feature_id
  839. LEFT JOIN {tripal_gffprotein_temp} TGPT on TGPT.parent_id = F.feature_id
  840. GROUP BY F.feature_id, F.name, F.uniquename, CVT.cvterm_id, CVT.name,
  841. TGPT.feature_id, TGPT.fmin, TGPT.fmax, TGCT.strand, FLM.uniquename
  842. ";
  843. $results = chado_query($sql);
  844. $protein_cvterm = chado_get_cvterm(array(
  845. 'name' => 'polypeptide',
  846. 'cv_id' => array(
  847. 'name' => 'sequence'
  848. )
  849. ));
  850. while ($result = $results->fetchObject()) {
  851. // If a protein exists with this same parent then don't add a new
  852. // protein.
  853. if (!$result->protein_id) {
  854. // Get details about this protein
  855. if ($re_mrna and $re_protein) {
  856. // We use a regex to generate protein name from mRNA name
  857. $uname = preg_replace("/$re_mrna/", $re_protein, $result->uniquename);
  858. $name = $result->name;
  859. }
  860. else {
  861. // No regex, use the default '-protein' suffix
  862. $uname = $result->uniquename . '-protein';
  863. $name = $result->name;
  864. }
  865. $values = array(
  866. 'parent_id' => $result->feature_id,
  867. 'fmin' => $result->fmin
  868. );
  869. $min_phase = chado_select_record('tripal_gffcds_temp', array('phase'), $values);
  870. $values = array(
  871. 'parent_id' => $result->feature_id,
  872. 'fmax' => $result->fmax
  873. );
  874. $max_phase = chado_select_record('tripal_gffcds_temp', array('phase'), $values);
  875. $pfmin = $result->fmin;
  876. $pfmax = $result->fmax;
  877. if ($result->strand == '-1') {
  878. $pfmax -= $max_phase[0]->phase;
  879. }
  880. else {
  881. $pfmin += $min_phase[0]->phase;
  882. }
  883. if ($skip_protein == 0){
  884. // Add the new protein record.
  885. $feature = $this->loadFeature($organism, $analysis_id,
  886. $protein_cvterm, $uname, $name, '', 'f', 'f', 1, 0);
  887. // Add the derives_from relationship.
  888. $cvterm = chado_get_cvterm(array('cvterm_id' => $result->cvterm_id));
  889. $this->loadDerivesFrom($feature, $cvterm,
  890. $result->uniquename, $organism, $pfmin, $pfmax);
  891. // Add the featureloc record. Set the start of the protein to
  892. // be the start of the coding sequence minus the phase.
  893. $this->loadFeatureLoc($feature, $organism, $result->landmark,
  894. $pfmin, $pfmax, $result->strand, '', 'f', 'f', '', 0);
  895. }
  896. }
  897. }
  898. }
  899. $this->logMessage("Setting ranks of children...");
  900. // Get features in a relationship that are also children of an alignment.
  901. $sql = "
  902. SELECT DISTINCT F.feature_id, F.organism_id, F.type_id,
  903. F.uniquename, FL.strand
  904. FROM {tripal_gff_temp} TGT
  905. INNER JOIN {feature} F ON TGT.feature_id = F.feature_id
  906. INNER JOIN {feature_relationship} FR ON FR.object_id = TGT.feature_id
  907. INNER JOIN {cvterm} CVT ON CVT.cvterm_id = FR.type_id
  908. INNER JOIN {featureloc} FL ON FL.feature_id = F.feature_id
  909. WHERE CVT.name = 'part_of'
  910. ";
  911. $parents = chado_query($sql);
  912. // Build and prepare the SQL for selecting the children relationship.
  913. $sel_gffchildren_sql = "
  914. SELECT DISTINCT FR.feature_relationship_id, FL.fmin, FR.rank
  915. FROM {feature_relationship} FR
  916. INNER JOIN {featureloc} FL on FL.feature_id = FR.subject_id
  917. INNER JOIN {cvterm} CVT on CVT.cvterm_id = FR.type_id
  918. WHERE FR.object_id = :feature_id AND CVT.name = 'part_of'
  919. ORDER BY FL.fmin ASC
  920. ";
  921. // Now set the rank of any parent/child relationships. The order is based
  922. // on the fmin. The start rank is 1. This allows features with other
  923. // relationships to be '0' (the default), and doesn't interfer with the
  924. // ordering defined here.
  925. $num_recs = $parents->rowCount();
  926. $i = 1;
  927. while ($parent = $parents->fetchObject()) {
  928. // get the children
  929. $result = chado_query($sel_gffchildren_sql, array(':feature_id' => $parent->feature_id));
  930. // build an array of the children
  931. $children = array();
  932. while ($child = $result->fetchObject()) {
  933. $children[] = $child;
  934. }
  935. // the children list comes sorted in ascending fmin
  936. // but if the parent is on the reverse strand we need to
  937. // reverse the order of the children.
  938. if ($parent->strand == -1) {
  939. arsort($children);
  940. }
  941. // first set the ranks to a negative number so that we don't
  942. // get a duplicate error message when we try to change any of them
  943. $rank = -1;
  944. foreach ($children as $child) {
  945. $match = array('feature_relationship_id' => $child->feature_relationship_id);
  946. $values = array('rank' => $rank);
  947. chado_update_record('feature_relationship', $match, $values);
  948. $rank--;
  949. }
  950. // now set the rank correctly. The rank should start at 0.
  951. $rank = 0;
  952. foreach ($children as $child) {
  953. $match = array('feature_relationship_id' => $child->feature_relationship_id);
  954. $values = array('rank' => $rank);
  955. chado_update_record('feature_relationship', $match, $values);
  956. $rank++;
  957. }
  958. $i++;
  959. }
  960. }
  961. return 1;
  962. }
  963. /**
  964. * Load the derives from attribute for a gff3 feature
  965. *
  966. * @param $feature
  967. * @param $subject
  968. * @param $organism
  969. *
  970. * @ingroup gff3_loader
  971. */
  972. private function loadDerivesFrom($feature, $cvterm, $object,
  973. $organism, $fmin, $fmax) {
  974. $type = $cvterm->name;
  975. // First look for the object feature in the temp table to get it's type.
  976. $values = array(
  977. 'organism_id' => $organism->organism_id,
  978. 'uniquename' => $object,
  979. );
  980. $result = chado_select_record('tripal_gff_temp', array('type_name'), $values);
  981. $type_id = NULL;
  982. if (count($result) > 0) {
  983. $otype = chado_get_cvterm(array(
  984. 'name' => $result[0]->type_name,
  985. 'cv_id' => array(
  986. 'name' => 'sequence'
  987. )
  988. ));
  989. if ($otype) {
  990. $type_id = $otype->cvterm_id;
  991. }
  992. }
  993. // If the object wasn't in the temp table then look for it in the
  994. // feature table and get it's type.
  995. if (!$type_id) {
  996. $result = chado_select_record('feature', array('type_id'), $values);
  997. if (count($result) > 1) {
  998. $this->logMessage("Cannot find feature type for, '!subject' , in 'derives_from' relationship. Multiple matching features exist with this uniquename.",
  999. array('!subject' => $object), TRIPAL_WARNING);
  1000. return;
  1001. }
  1002. else if (count($result) == 0) {
  1003. $this->logMessage("Cannot find feature type for, '!subject' , in 'derives_from' relationship.",
  1004. array('!subject' => $object), TRIPAL_WARNING);
  1005. return '';
  1006. }
  1007. else {
  1008. $type_id = $result->type_id;
  1009. }
  1010. }
  1011. // Get the object feature.
  1012. $match = array(
  1013. 'organism_id' => $organism->organism_id,
  1014. 'uniquename' => $object,
  1015. 'type_id' => $type_id,
  1016. );
  1017. $ofeature = chado_select_record('feature', array('feature_id'), $match);
  1018. if (count($ofeature) == 0) {
  1019. $this->logMessage("Could not add 'Derives_from' relationship " .
  1020. "for %uniquename and %subject. Subject feature, '%subject', " .
  1021. "cannot be found.", array('%uniquename' => $feature->uniquename, '%subject' => $subject), TRIPAL_ERROR);
  1022. return;
  1023. }
  1024. // If this feature is a protein then add it to the tripal_gffprotein_temp.
  1025. if ($type == 'protein' or $type == 'polypeptide') {
  1026. $values = array(
  1027. 'feature_id' => $feature->feature_id,
  1028. 'parent_id' => $ofeature[0]->feature_id,
  1029. 'fmin' => $fmin,
  1030. 'fmax' => $fmax
  1031. );
  1032. $result = chado_insert_record('tripal_gffprotein_temp', $values);
  1033. if (!$result) {
  1034. throw new Exception(t("Cound not save record in temporary protein table, Cannot continue.", array()));
  1035. }
  1036. }
  1037. // Now check to see if the relationship already exists. If it does
  1038. // then just return.
  1039. $values = array(
  1040. 'object_id' => $ofeature[0]->feature_id,
  1041. 'subject_id' => $feature->feature_id,
  1042. 'type_id' => array(
  1043. 'cv_id' => array(
  1044. 'name' => 'sequence'
  1045. ),
  1046. 'name' => 'derives_from',
  1047. ),
  1048. 'rank' => 0
  1049. );
  1050. $rel = chado_select_record('feature_relationship', array('*'), $values);
  1051. if (count($rel) > 0) {
  1052. return;
  1053. }
  1054. // finally insert the relationship if it doesn't exist
  1055. $ret = chado_insert_record('feature_relationship', $values);
  1056. if (!$ret) {
  1057. $this->logMessage("Could not add 'Derives_from' relationship for :uniquename and :subject.",
  1058. array(':uniquename' => $feature->uniquename, ':subject' => $subject), TRIPAL_WARNING);
  1059. }
  1060. }
  1061. /**
  1062. * Load the parents for a gff3 feature
  1063. *
  1064. * @param $feature
  1065. * @param $cvterm
  1066. * @param $parents
  1067. * @param $organism_id
  1068. * @param $fmin
  1069. *
  1070. * @ingroup gff3_loader
  1071. */
  1072. private function loadParents($feature, $cvterm, $parents,
  1073. $organism_id, $strand, $phase, $fmin, $fmax) {
  1074. $uname = $feature->uniquename;
  1075. $type = $cvterm->name;
  1076. $rel_type = 'part_of';
  1077. // Prepare these SQL statements that will be used repeatedly.
  1078. $cvterm_sql = "
  1079. SELECT CVT.cvterm_id
  1080. FROM {cvterm} CVT
  1081. INNER JOIN {cv} CV on CVT.cv_id = CV.cv_id
  1082. LEFT JOIN {cvtermsynonym} CVTS on CVTS.cvterm_id = CVT.cvterm_id
  1083. WHERE cv.name = :cvname and (CVT.name = :name or CVTS.synonym = :synonym)
  1084. ";
  1085. // Iterate through the parents in the list.
  1086. foreach ($parents as $parent) {
  1087. // Get the parent cvterm.
  1088. $values = array(
  1089. 'organism_id' => $organism_id,
  1090. 'uniquename' => $parent,
  1091. );
  1092. $result = chado_select_record('tripal_gff_temp', array('type_name'), $values);
  1093. if (count($result) == 0) {
  1094. $this->logMessage("Cannot find parent: %parent.", array('%parent' => $parent), TRIPAL_WARNING);
  1095. return '';
  1096. }
  1097. $parent_type = $result[0]->type_name;
  1098. // try to find the parent
  1099. $parentcvterm = chado_query($cvterm_sql, array(':cvname' => 'sequence', ':name' => $parent_type, ':synonym' => $parent_type))->fetchObject();
  1100. $relcvterm = chado_query($cvterm_sql, array(':cvname' => 'sequence', ':name' => $rel_type, ':synonym' => $rel_type))->fetchObject();
  1101. if (!$relcvterm) {
  1102. throw new Exception(t("Cannot find the term, 'part_of', from the sequence ontology. This term is used for associating parent and children features. Please check that the ontology is fully imported."));
  1103. }
  1104. $values = array(
  1105. 'organism_id' => $organism_id,
  1106. 'uniquename' => $parent,
  1107. 'type_id' => $parentcvterm->cvterm_id,
  1108. );
  1109. $result = chado_select_record('feature', array('feature_id'), $values);
  1110. $parent_feature = $result[0];
  1111. // if the parent exists then add the relationship otherwise print error and skip
  1112. if ($parent_feature) {
  1113. // check to see if the relationship already exists
  1114. $values = array(
  1115. 'object_id' => $parent_feature->feature_id,
  1116. 'subject_id' => $feature->feature_id,
  1117. 'type_id' => $relcvterm->cvterm_id,
  1118. );
  1119. $rel = chado_select_record('feature_relationship', array('*'), $values);
  1120. if (count($rel) > 0) {
  1121. }
  1122. else {
  1123. // the relationship doesn't already exist, so add it.
  1124. $values = array(
  1125. 'subject_id' => $feature->feature_id,
  1126. 'object_id' => $parent_feature->feature_id,
  1127. 'type_id' => $relcvterm->cvterm_id,
  1128. );
  1129. $result = chado_insert_record('feature_relationship', $values);
  1130. if (!$result) {
  1131. $this->logMessage("Failed to insert feature relationship '$uname' ($type) $rel_type '$parent' ($parent_type).",
  1132. array(), TRIPAL_WARNING);
  1133. }
  1134. }
  1135. // If this feature is a CDS and now that we know the parent we can
  1136. // add it to the tripal_gffcds_temp table for later lookup.
  1137. if ($type == 'CDS') {
  1138. $values = array(
  1139. 'feature_id' => $feature->feature_id,
  1140. 'parent_id' => $parent_feature->feature_id,
  1141. 'fmin' => $fmin,
  1142. 'fmax' => $fmax,
  1143. 'strand' => $strand,
  1144. );
  1145. if (isset($phase)) {
  1146. $values['phase'] = $phase;
  1147. }
  1148. $result = chado_insert_record('tripal_gffcds_temp', $values);
  1149. if (!$result) {
  1150. throw new Exception(t("Cound not save record in temporary CDS table, Cannot continue.", array()));
  1151. exit;
  1152. }
  1153. }
  1154. }
  1155. else {
  1156. $this->logMessage("Cannot establish relationship '$uname' ($type) $rel_type '$parent' ($parent_type): Cannot find the parent.",
  1157. array(), TRIPAL_WARNING);
  1158. }
  1159. }
  1160. }
  1161. /**
  1162. * Load the dbxref attribute for a feature
  1163. *
  1164. * @param $feature
  1165. * @param $dbxrefs
  1166. *
  1167. * @ingroup gff3_loader
  1168. */
  1169. private function loadDbxref($feature, $dbxrefs) {
  1170. // iterate through each of the dbxrefs
  1171. foreach ($dbxrefs as $dbxref) {
  1172. // get the database name from the reference. If it doesn't exist then create one.
  1173. $ref = explode(":", $dbxref);
  1174. $dbname = trim($ref[0]);
  1175. $accession = trim($ref[1]);
  1176. // first look for the database name if it doesn't exist then create one.
  1177. // first check for the fully qualified URI (e.g. DB:<dbname>. If that
  1178. // can't be found then look for the name as is. If it still can't be found
  1179. // the create the database
  1180. $values = array('name' => "DB:$dbname");
  1181. $db = chado_select_record('db', array('db_id'), $values);
  1182. if (count($db) == 0) {
  1183. $values = array('name' => "$dbname");
  1184. $db = chado_select_record('db', array('db_id'), $values);
  1185. }
  1186. if (count($db) == 0) {
  1187. $values = array(
  1188. 'name' => $dbname,
  1189. 'description' => 'Added automatically by the GFF loader'
  1190. );
  1191. $success = chado_insert_record('db', $values);
  1192. if ($success) {
  1193. $values = array('name' => "$dbname");
  1194. $db = chado_select_record('db', array('db_id'), $values);
  1195. }
  1196. else {
  1197. $this->logMessage("Cannot find or add the database $dbname.", array(), TRIPAL_WARNING);
  1198. return 0;
  1199. }
  1200. }
  1201. $db = $db[0];
  1202. // now check to see if the accession exists
  1203. $values = array(
  1204. 'accession' => $accession,
  1205. 'db_id' => $db->db_id
  1206. );
  1207. $dbxref = chado_select_record('dbxref', array('dbxref_id'), $values);
  1208. // if the accession doesn't exist then we want to add it
  1209. if (sizeof($dbxref) == 0) {
  1210. $values = array(
  1211. 'db_id' => $db->db_id,
  1212. 'accession' => $accession,
  1213. 'version' => ''
  1214. );
  1215. $ret = chado_insert_record('dbxref', $values);
  1216. $values = array(
  1217. 'accession' => $accession,
  1218. 'db_id' => $db->db_id
  1219. );
  1220. $dbxref = chado_select_record('dbxref', array('dbxref_id'), $values);
  1221. }
  1222. $dbxref = $dbxref[0];
  1223. // check to see if this feature dbxref already exists
  1224. $values = array(
  1225. 'dbxref_id' => $dbxref->dbxref_id,
  1226. 'feature_id' => $feature->feature_id
  1227. );
  1228. $fdbx = chado_select_record('feature_dbxref', array('feature_dbxref_id'), $values);
  1229. // now associate this feature with the database reference if it doesn't
  1230. // already exist
  1231. if (sizeof($fdbx) == 0) {
  1232. $values = array(
  1233. 'dbxref_id' => $dbxref->dbxref_id,
  1234. 'feature_id' => $feature->feature_id
  1235. );
  1236. $success = chado_insert_record('feature_dbxref', $values);
  1237. if (!$success) {
  1238. $this->logMessage("Failed to insert Dbxref: $dbname:$accession.", array(), TRIPAL_WARNING);
  1239. return 0;
  1240. }
  1241. }
  1242. }
  1243. return 1;
  1244. }
  1245. /**
  1246. * Load the cvterms for a feature.
  1247. *
  1248. * Assumes there is a dbxref.accession matching a cvterm.name
  1249. *
  1250. * @param $feature
  1251. * @param $dbxrefs
  1252. *
  1253. * @ingroup gff3_loader
  1254. */
  1255. private function loadOntology($feature, $dbxrefs) {
  1256. // iterate through each of the dbxrefs
  1257. foreach ($dbxrefs as $dbxref) {
  1258. // get the database name from the reference. If it doesn't exist then create one.
  1259. $ref = explode(":", $dbxref);
  1260. $dbname = trim($ref[0]);
  1261. $accession = trim($ref[1]);
  1262. // first look for the database name
  1263. $db = chado_select_record('db', array('db_id'), array('name' => "DB:$dbname"));
  1264. if (sizeof($db) == 0) {
  1265. // now look for the name without the 'DB:' prefix.
  1266. $db = chado_select_record('db', array('db_id'), array('name' => "$dbname"));
  1267. if (sizeof($db) == 0) {
  1268. $this->logMessage("Database, $dbname, is not present. Cannot associate term: $dbname:$accession.", array(), TRIPAL_WARNING);
  1269. return 0;
  1270. }
  1271. }
  1272. $db = $db[0];
  1273. // now check to see if the accession exists
  1274. $dbxref = chado_select_record('dbxref', array('dbxref_id'),
  1275. array('accession' => $accession, 'db_id' => $db->db_id));
  1276. if (sizeof($dbxref) == 0) {
  1277. $this->logMessage("Accession, $accession is missing for reference: $dbname:$accession.", array(), TRIPAL_WARNING);
  1278. return 0;
  1279. }
  1280. $dbxref = $dbxref[0];
  1281. // now check to see if the cvterm exists
  1282. $cvterm = chado_select_record('cvterm', array('cvterm_id'), array(
  1283. 'dbxref_id' => $dbxref->dbxref_id));
  1284. // if it doesn't exist in the cvterm table, look for an alternate id
  1285. if (sizeof($cvterm) == 0) {
  1286. $cvterm = chado_select_record('cvterm_dbxref', array('cvterm_id'), array(
  1287. 'dbxref_id' => $dbxref->dbxref_id));
  1288. if (sizeof($cvterm) == 0) {
  1289. $this->logMessage("CV Term is missing for reference: $dbname:$accession.", array(), TRIPAL_WARNING);
  1290. return 0;
  1291. }
  1292. }
  1293. $cvterm = $cvterm[0];
  1294. // check to see if this feature cvterm already exists
  1295. $fcvt = chado_select_record('feature_cvterm', array('feature_cvterm_id'),
  1296. array('cvterm_id' => $cvterm->cvterm_id, 'feature_id' => $feature->feature_id));
  1297. // now associate this feature with the cvterm if it doesn't already exist
  1298. if (sizeof($fcvt)==0) {
  1299. $values = array(
  1300. 'cvterm_id' => $cvterm->cvterm_id,
  1301. 'feature_id' => $feature->feature_id,
  1302. 'pub_id' => array(
  1303. 'uniquename' => 'null',
  1304. ),
  1305. );
  1306. $success = chado_insert_record('feature_cvterm', $values);
  1307. if (!$success) {
  1308. $this->logMessage("Failed to insert ontology term: $dbname:$accession.", array(), TRIPAL_WARNING);
  1309. return 0;
  1310. }
  1311. }
  1312. }
  1313. return 1;
  1314. }
  1315. /**
  1316. * Load any aliases for a feature
  1317. *
  1318. * @param $feature
  1319. * @param $aliases
  1320. *
  1321. * @ingroup gff3_loader
  1322. */
  1323. private function loadAlias($feature, $aliases) {
  1324. // make sure we have a 'synonym_type' vocabulary
  1325. $select = array('name' => 'synonym_type');
  1326. $results = chado_select_record('cv', array('*'), $select);
  1327. if (count($results) == 0) {
  1328. // insert the 'synonym_type' vocabulary
  1329. $values = array(
  1330. 'name' => 'synonym_type',
  1331. 'definition' => 'vocabulary for synonym types',
  1332. );
  1333. $success = chado_insert_record('cv', $values);
  1334. if (!$success) {
  1335. $this->logMessage("Failed to add the synonyms type vocabulary.", array(), TRIPAL_WARNING);
  1336. return 0;
  1337. }
  1338. // now that we've added the cv we need to get the record
  1339. $results = chado_select_record('cv', array('*'), $select);
  1340. if (count($results) > 0) {
  1341. $syncv = $results[0];
  1342. }
  1343. }
  1344. else {
  1345. $syncv = $results[0];
  1346. }
  1347. // get the 'exact' cvterm, which is the type of synonym we're adding
  1348. $select = array(
  1349. 'name' => 'exact',
  1350. 'cv_id' => array(
  1351. 'name' => 'synonym_type'
  1352. ),
  1353. );
  1354. $result = chado_select_record('cvterm', array('*'), $select);
  1355. if (count($result) == 0) {
  1356. $term = array(
  1357. 'name' => 'exact',
  1358. 'id' => "synonym_type:exact",
  1359. 'definition' => '',
  1360. 'is_obsolete' => 0,
  1361. 'cv_name' => $syncv->name,
  1362. 'is_relationship' => FALSE
  1363. );
  1364. $syntype = chado_insert_cvterm($term, array('update_existing' => TRUE));
  1365. if (!$syntype) {
  1366. $this->logMessage("Cannot add synonym type: internal:$type.", array(), TRIPAL_WARNING);
  1367. return 0;
  1368. }
  1369. }
  1370. else {
  1371. $syntype = $result[0];
  1372. }
  1373. // iterate through all of the aliases and add each one
  1374. foreach ($aliases as $alias) {
  1375. // check to see if the alias already exists in the synonym table
  1376. // if not, then add it
  1377. $select = array(
  1378. 'name' => $alias,
  1379. 'type_id' => $syntype->cvterm_id,
  1380. );
  1381. $result = chado_select_record('synonym', array('*'), $select);
  1382. if (count($result) == 0) {
  1383. $values = array(
  1384. 'name' => $alias,
  1385. 'type_id' => $syntype->cvterm_id,
  1386. 'synonym_sgml' => '',
  1387. );
  1388. $success = chado_insert_record('synonym', $values);
  1389. if (!$success) {
  1390. $this->logMessage("Cannot add alias $alias to synonym table.", array(), TRIPAL_WARNING);
  1391. return 0;
  1392. }
  1393. $result = chado_select_record('synonym', array('*'), $select);
  1394. $synonym = $result[0];
  1395. }
  1396. else {
  1397. $synonym = $result[0];
  1398. }
  1399. // check to see if we have a NULL publication in the pub table. If not,
  1400. // then add one.
  1401. $select = array('uniquename' => 'null');
  1402. $result = chado_select_record('pub', array('*'), $select);
  1403. if (count($result) == 0) {
  1404. $pub_sql = "
  1405. INSERT INTO {pub} (uniquename,type_id)
  1406. VALUES (:uname,
  1407. (SELECT cvterm_id
  1408. FROM {cvterm} CVT
  1409. INNER JOIN {dbxref} DBX ON DBX.dbxref_id = CVT.dbxref_id
  1410. INNER JOIN {db} DB ON DB.db_id = DBX.db_id
  1411. WHERE CVT.name = :type_id))
  1412. ";
  1413. $status = chado_query($psql);
  1414. if (!$status) {
  1415. $this->logMessage("Cannot prepare statement 'ins_pub_uniquename_typeid.", array(), TRIPAL_WARNING);
  1416. return 0;
  1417. }
  1418. // insert the null pub
  1419. $result = chado_query($pub_sql, array(':uname' => 'null', ':type_id' => 'null'))->fetchObject();
  1420. if (!$result) {
  1421. $this->logMessage("Cannot add null publication needed for setup of alias.", array(), TRIPAL_WARNING);
  1422. return 0;
  1423. }
  1424. $result = chado_select_record('pub', array('*'), $select);
  1425. $pub = $result[0];
  1426. }
  1427. else {
  1428. $pub = $result[0];
  1429. }
  1430. // check to see if the synonym exists in the feature_synonym table
  1431. // if not, then add it.
  1432. $values = array(
  1433. 'synonym_id' => $synonym->synonym_id,
  1434. 'feature_id' => $feature->feature_id,
  1435. 'pub_id' => $pub->pub_id,
  1436. );
  1437. $columns = array('feature_synonym_id');
  1438. $result = chado_select_record('feature_synonym', $columns, $values);
  1439. if (count($result) == 0) {
  1440. $values = array(
  1441. 'synonym_id' => $synonym->synonym_id,
  1442. 'feature_id' => $feature->feature_id,
  1443. 'pub_id' => $pub->pub_id,
  1444. );
  1445. $success = chado_insert_record('feature_synonym', $values);
  1446. if (!$success) {
  1447. $this->logMessage("Cannot add alias $alias to feature synonym table.", array(), TRIPAL_WARNING);
  1448. return 0;
  1449. }
  1450. }
  1451. }
  1452. return 1;
  1453. }
  1454. /**
  1455. * Create the feature record & link it to it's analysis
  1456. *
  1457. * @param $organism
  1458. * @param $analysis_id
  1459. * @param $cvterm
  1460. * @param $uniquename
  1461. * @param $name
  1462. * @param $residues
  1463. * @param $is_analysis
  1464. * @param $is_obsolete
  1465. * @param $add_only
  1466. * @param $score
  1467. *
  1468. * @ingroup gff3_loader
  1469. */
  1470. private function loadFeature($organism, $analysis_id, $cvterm, $uniquename,
  1471. $name, $residues, $is_analysis = 'f', $is_obsolete = 'f', $add_only, $score) {
  1472. // Check to see if the feature already exists.
  1473. $feature = NULL;
  1474. $fselect = array(
  1475. 'organism_id' => $organism->organism_id,
  1476. 'uniquename' => $uniquename,
  1477. 'type_id' => $cvterm->cvterm_id
  1478. );
  1479. $columns = array('feature_id', 'name', 'uniquename', 'seqlen', 'organism_id', 'type_id');
  1480. $result = chado_select_record('feature', $columns, $fselect);
  1481. if (count($result) > 0) {
  1482. $feature = $result[0];
  1483. }
  1484. if (strcmp($is_obsolete, 'f')==0 or $is_obsolete == 0) {
  1485. $is_obsolete = 'FALSE';
  1486. }
  1487. if (strcmp($is_obsolete, 't')==0 or $is_obsolete == 1) {
  1488. $is_obsolete = 'TRUE';
  1489. }
  1490. if (strcmp($is_analysis, 'f')==0 or $is_analysis == 0) {
  1491. $is_analysis = 'FALSE';
  1492. }
  1493. if (strcmp($is_analysis, 't')==0 or $is_analysis == 1) {
  1494. $is_analysis = 'TRUE';
  1495. }
  1496. // Insert the feature if it does not exist otherwise perform an update.
  1497. if (!$feature) {
  1498. $values = array(
  1499. 'organism_id' => $organism->organism_id,
  1500. 'name' => $name,
  1501. 'uniquename' => $uniquename,
  1502. 'md5checksum' => md5($residues),
  1503. 'type_id' => $cvterm->cvterm_id,
  1504. 'is_analysis' => $is_analysis,
  1505. 'is_obsolete' => $is_obsolete,
  1506. );
  1507. $feature = (object) chado_insert_record('feature', $values);
  1508. if (!$feature) {
  1509. $this->logMessage("Failed to insert feature '$uniquename' ($cvterm->name).", array(), TRIPAL_WARNING);
  1510. return 0;
  1511. }
  1512. }
  1513. elseif (!$add_only) {
  1514. $values = array(
  1515. 'name' => $name,
  1516. 'md5checksum' => md5($residues),
  1517. 'is_analysis' => $is_analysis,
  1518. 'is_obsolete' => $is_obsolete,
  1519. );
  1520. $match = array(
  1521. 'organism_id' => $organism->organism_id,
  1522. 'uniquename' => $uniquename,
  1523. 'type_id' => $cvterm->cvterm_id,
  1524. );
  1525. $result = chado_update_record('feature', $match, $values);
  1526. if (!$result) {
  1527. $this->logMessage("Failed to update feature '$uniquename' ($cvterm->name).", array(), TRIPAL_WARNING);
  1528. return 0;
  1529. }
  1530. }
  1531. else {
  1532. // The feature exists and we don't want to update it so return
  1533. // a value of 0. This will stop all downstream property additions
  1534. return $feature;
  1535. }
  1536. // Add the analysisfeature entry to the analysisfeature table if
  1537. // it doesn't already exist.
  1538. $af_values = array(
  1539. 'analysis_id' => $analysis_id,
  1540. 'feature_id' => $feature->feature_id
  1541. );
  1542. $afeature = chado_select_record('analysisfeature', array('analysisfeature_id'), $af_values);
  1543. if (count($afeature)==0) {
  1544. // if a score is available then set that to be the significance field
  1545. if (strcmp($score, '.') != 0) {
  1546. $af_values['significance'] = $score;
  1547. }
  1548. if (!chado_insert_record('analysisfeature', $af_values)) {
  1549. $this->logMessage("Could not add analysisfeature record: $analysis_id, $feature->feature_id.", array(), TRIPAL_WARNING);
  1550. }
  1551. }
  1552. else {
  1553. // if a score is available then set that to be the significance field
  1554. $new_vals = array();
  1555. if (strcmp($score, '.')!=0) {
  1556. $new_vals['significance'] = $score;
  1557. }
  1558. else {
  1559. $new_vals['significance'] = '__NULL__';
  1560. }
  1561. if (!$add_only) {
  1562. $ret = chado_update_record('analysisfeature', $af_values, $new_vals);
  1563. if (!$ret) {
  1564. $this->logMessage("Could not update analysisfeature record: $analysis_id, $feature->feature_id.", array(), TRIPAL_WARNING);
  1565. }
  1566. }
  1567. }
  1568. return $feature;
  1569. }
  1570. /**
  1571. * Insert the location of the feature
  1572. *
  1573. * @param $feature
  1574. * @param $organism
  1575. * @param $landmark
  1576. * @param $fmin
  1577. * @param $fmax
  1578. * @param $strand
  1579. * @param $phase
  1580. * @param $is_fmin_partial
  1581. * @param $is_fmax_partial
  1582. * @param $residue_info
  1583. * @param $locgroup
  1584. * @param $landmark_type_id
  1585. * @param $landmark_organism_id
  1586. * @param $create_landmark
  1587. * @param $landmark_is_target
  1588. *
  1589. * @ingroup gff3_loader
  1590. */
  1591. private function loadFeatureLoc($feature, $organism, $landmark, $fmin,
  1592. $fmax, $strand, $phase, $is_fmin_partial, $is_fmax_partial, $residue_info, $locgroup,
  1593. $landmark_type_id = '', $landmark_organism_id = '', $create_landmark = 0,
  1594. $landmark_is_target = 0) {
  1595. $select = array(
  1596. 'organism_id' => $landmark_organism_id ? $landmark_organism_id : $organism->organism_id,
  1597. 'uniquename' => $landmark,
  1598. );
  1599. if ($landmark_type_id) {
  1600. $select['type_id'] = $landmark_type_id;
  1601. }
  1602. $results = chado_select_record('feature', array('feature_id'), $select);
  1603. $srcfeature = '';
  1604. if (count($results)==0) {
  1605. // so we couldn't find the landmark using the uniquename. Let's try the 'name'.
  1606. // if we return only a single result then we can proceed. Otherwise give an
  1607. $select = array(
  1608. 'organism_id' => $landmark_organism_id ? $landmark_organism_id : $organism->organism_id,
  1609. 'name' => $landmark,
  1610. );
  1611. if ($landmark_type_id) {
  1612. $select['type_id'] = $landmark_type_id;
  1613. }
  1614. $results = chado_select_record('feature', array('feature_id'), $select);
  1615. if (count($results) == 0) {
  1616. // if the landmark is the target feature in a matched alignment then try one more time to
  1617. // find it by querying any feature with the same uniquename. If we find one then use it.
  1618. if ($landmark_is_target) {
  1619. $select = array('uniquename' => $landmark);
  1620. $results = chado_select_record('feature', array('feature_id'), $select);
  1621. if (count($results) == 1) {
  1622. $srcfeature = $results[0];
  1623. }
  1624. }
  1625. if (!$srcfeature) {
  1626. // we couldn't find the landmark feature, so if the user has requested we create it then do so
  1627. // but only if we have a type id
  1628. if ($create_landmark and $landmark_type_id) {
  1629. $values = array(
  1630. 'organism_id' => $landmark_organism_id ? $landmark_organism_id : $organism->organism_id,
  1631. 'name' => $landmark,
  1632. 'uniquename' => $landmark,
  1633. 'type_id' => $landmark_type_id
  1634. );
  1635. $results = chado_insert_record('feature', $values);
  1636. if (!$results) {
  1637. $this->logMessage("Cannot find landmark feature: '%landmark', nor could it be inserted.",
  1638. array('%landmark' => $landmark), TRIPAL_WARNING);
  1639. return 0;
  1640. }
  1641. $srcfeature = new stdClass();
  1642. $srcfeature->feature_id = $results['feature_id'];
  1643. }
  1644. else {
  1645. $this->logMessage("Cannot find unique landmark feature: '%landmark'.",
  1646. array('%landmark' => $landmark), TRIPAL_WARNING);
  1647. return 0;
  1648. }
  1649. }
  1650. }
  1651. elseif (count($results) > 1) {
  1652. $this->logMessage("multiple landmarks exist with the name: '%landmark'. Cannot " .
  1653. "resolve which one to use. Cannot add the feature location record.",
  1654. array('%landmark' => $landmark), TRIPAL_WARNING);
  1655. return 0;
  1656. }
  1657. else {
  1658. $srcfeature = $results[0];
  1659. }
  1660. }
  1661. elseif (count($results) > 1) {
  1662. $this->logMessage("multiple landmarks exist with the name: '%landmark'. Cannot " .
  1663. "resolve which one to use. Cannot add the feature location record.",
  1664. array('%landmark' => $landmark, TRIPAL_WARNING));
  1665. return 0;
  1666. }
  1667. else {
  1668. $srcfeature = $results[0];
  1669. }
  1670. // TODO: create an attribute that recognizes the residue_info,locgroup,
  1671. // is_fmin_partial and is_fmax_partial, right now these are
  1672. // hardcoded to be false and 0 below.
  1673. // check to see if this featureloc already exists, but also keep track of the
  1674. // last rank value
  1675. $rank = 0;
  1676. $exists = 0;
  1677. $select = array('feature_id' => $feature->feature_id);
  1678. $options = array(
  1679. 'order_by' => array(
  1680. 'rank' => 'ASC'
  1681. ),
  1682. );
  1683. $locrecs = chado_select_record('featureloc', array('*'), $select, $options);
  1684. foreach ($locrecs as $featureloc) {
  1685. // it is possible for the featureloc->srcfeature_id to be NULL. This can happen if the srcfeature
  1686. // is not known (according to chado table field descriptions). If it's null then just skip this entry
  1687. if (!$featureloc->srcfeature_id) {
  1688. continue;
  1689. }
  1690. $select = array('feature_id' => $featureloc->srcfeature_id);
  1691. $columns = array('feature_id', 'name');
  1692. $locsfeature = chado_select_record('feature', $columns, $select);
  1693. // the source feature name and at least the fmin and fmax must be the same
  1694. // for an update of the featureloc, otherwise we'll insert a new record.
  1695. if (strcmp($locsfeature[0]->name, $landmark)==0 and
  1696. ($featureloc->fmin == $fmin or $featureloc->fmax == $fmax)) {
  1697. $match = array('featureloc_id' => $featureloc->featureloc_id);
  1698. $values = array();
  1699. $exists = 1;
  1700. if ($featureloc->fmin != $fmin) {
  1701. $values['fmin'] = $fmin;
  1702. }
  1703. if ($featureloc->fmax != $fmax) {
  1704. $values['fmax'] = $fmax;
  1705. }
  1706. if ($featureloc->strand != $strand) {
  1707. $values['strand'] = $strand;
  1708. }
  1709. if (count($values) > 0) {
  1710. chado_update_record('featureloc', $match, $values);
  1711. }
  1712. }
  1713. $rank = $featureloc->rank + 1;
  1714. }
  1715. if (!$exists) {
  1716. // this feature location is new so add it
  1717. if (strcmp($is_fmin_partial, 'f')==0 or !$is_fmin_partial) {
  1718. $is_fmin_partial = 'FALSE';
  1719. }
  1720. elseif (strcmp($is_fmin_partial, 't')==0 or $is_fmin_partial = 1) {
  1721. $is_fmin_partial = 'TRUE';
  1722. }
  1723. if (strcmp($is_fmax_partial, 'f')==0 or !$is_fmax_partial) {
  1724. $is_fmax_partial = 'FALSE';
  1725. }
  1726. elseif (strcmp($is_fmax_partial, 't')==0 or $is_fmax_partial = 1) {
  1727. $is_fmax_partial = 'TRUE';
  1728. }
  1729. $values = array(
  1730. 'feature_id' => $feature->feature_id,
  1731. 'srcfeature_id' => $srcfeature->feature_id,
  1732. 'fmin' => $fmin,
  1733. 'is_fmin_partial' => $is_fmin_partial,
  1734. 'fmax' => $fmax,
  1735. 'is_fmax_partial' => $is_fmax_partial,
  1736. 'strand' => $strand,
  1737. 'residue_info' => $residue_info,
  1738. 'locgroup' => $locgroup,
  1739. 'rank' => $rank
  1740. );
  1741. if ($phase) {
  1742. $values['phase'] = $phase;
  1743. }
  1744. $success = chado_insert_record('featureloc', $values);
  1745. if (!$success) {
  1746. throw new Exception("Failed to insert featureloc.");
  1747. }
  1748. }
  1749. return 1;
  1750. }
  1751. /**
  1752. * Load a preoprty (featurepop) for the feature
  1753. *
  1754. * @param $feature
  1755. * @param $property
  1756. * @param $value
  1757. *
  1758. * @ingroup gff3_loader
  1759. */
  1760. private function loadProperty($feature, $property, $value) {
  1761. // first make sure the cvterm exists. if not, then add it
  1762. $select = array(
  1763. 'name' => $property,
  1764. 'cv_id' => array(
  1765. 'name' => 'feature_property',
  1766. ),
  1767. );
  1768. $result = chado_select_record('cvterm', array('*'), $select);
  1769. // if we don't have a property like this already, then add it otherwise, just return
  1770. if (count($result) == 0) {
  1771. $term = array(
  1772. 'id' => "null:$property",
  1773. 'name' => $property,
  1774. 'namespace' => 'feature_property',
  1775. 'is_obsolete' => 0,
  1776. 'cv_name' => 'feature_property',
  1777. 'is_relationship' => FALSE
  1778. );
  1779. $cvterm = (object)chado_insert_cvterm($term, array('update_existing' => FALSE));
  1780. if (!$cvterm) {
  1781. $this->logMessage("Cannot add cvterm, $property.", array(), TRIPAL_WARNING);
  1782. return 0;
  1783. }
  1784. }
  1785. else {
  1786. $cvterm = $result[0];
  1787. }
  1788. // check to see if the property already exists for this feature
  1789. // if it does but the value is unique then increment the rank and add it.
  1790. // if the value is not unique then don't add it.
  1791. $add = 1;
  1792. $rank = 0;
  1793. $select = array(
  1794. 'feature_id' => $feature->feature_id,
  1795. 'type_id' => $cvterm->cvterm_id,
  1796. );
  1797. $options = array(
  1798. 'order_by' => array(
  1799. 'rank' => 'ASC',
  1800. ),
  1801. );
  1802. $results = chado_select_record('featureprop', array('*'), $select, $options);
  1803. foreach ($results as $prop) {
  1804. if (strcmp($prop->value, $value)==0) {
  1805. $add = NULL; // don't add it, it already exists
  1806. }
  1807. $rank = $prop->rank + 1;
  1808. }
  1809. // add the property if we pass the check above
  1810. if ($add) {
  1811. $values = array(
  1812. 'feature_id' => $feature->feature_id,
  1813. 'type_id' => $cvterm->cvterm_id,
  1814. 'value' => $value,
  1815. 'rank' => $rank,
  1816. );
  1817. $result = chado_insert_record('featureprop', $values);
  1818. if (!$result) {
  1819. $this->logMessage("cannot add featureprop, $property.", array(), TRIPAL_WARNING);
  1820. }
  1821. }
  1822. }
  1823. /**
  1824. * Load the FASTA sequences at the bottom of a GFF3 file
  1825. *
  1826. * @param $fh
  1827. * @param $interval
  1828. * @param $num_read
  1829. * @param $line_num
  1830. * @param $filesize
  1831. *
  1832. * @ingroup gff3_loader
  1833. */
  1834. private function loadFasta($fh, $interval, &$num_read, &$line_num, $filesize) {
  1835. $this->logMessage("Loading FASTA sequences...");
  1836. $residues = '';
  1837. $id = NULL;
  1838. // iterate through the remaining lines of the file
  1839. while ($line = fgets($fh)) {
  1840. $line_num++;
  1841. $size = drupal_strlen($line);
  1842. $this->addItemsHandled($size);
  1843. $num_read += $size;
  1844. $line = trim($line);
  1845. // if we encounter a definition line then get the name, uniquename,
  1846. // accession and relationship subject from the definition line
  1847. if (preg_match('/^>/', $line)) {
  1848. // if we are beginning a new sequence then save to the database the last one we just finished.
  1849. if ($id) {
  1850. $values = array('uniquename' => $id);
  1851. $result = chado_select_record('tripal_gff_temp', array('*'), $values);
  1852. if (count($result) == 0) {
  1853. $this->logMessage('Cannot find feature to assign FASTA sequence: %uname.',
  1854. array('%uname' => $id), TRIPAL_WARNING);
  1855. }
  1856. else {
  1857. // if we have a feature then add the residues
  1858. $feature = $result[0];
  1859. $values = array(
  1860. 'residues' => $residues,
  1861. 'seqlen' => strlen($residues)
  1862. );
  1863. $match = array('feature_id' => $feature->feature_id);
  1864. chado_update_record('feature', $match, $values);
  1865. }
  1866. }
  1867. // get the feature ID for this ID from the tripal_gff_temp table. It
  1868. // should be the name up to the first space
  1869. $id = preg_replace('/^>([^\s]+).*$/', '\1', $line);
  1870. $residues = '';
  1871. }
  1872. else {
  1873. $residues .= trim($line);
  1874. }
  1875. }
  1876. // add in the last sequence
  1877. $values = array('uniquename' => $id);
  1878. $result = chado_select_record('tripal_gff_temp', array('*'), $values);
  1879. if (count($result) == 0) {
  1880. $this->logMessage('Cannot find feature to assign FASTA sequence: %uname.',
  1881. array('%uname' => $id), TRIPAL_WARNING);
  1882. }
  1883. else {
  1884. // if we have a feature then add the residues
  1885. $feature = $result[0];
  1886. $values = array(
  1887. 'residues' => $residues,
  1888. 'seqlen' => strlen($residues)
  1889. );
  1890. $match = array('feature_id' => $feature->feature_id);
  1891. chado_update_record('feature', $match, $values);
  1892. }
  1893. }
  1894. /**
  1895. * Load the target attribute of a gff3 record
  1896. *
  1897. * @param $feature
  1898. * @param $tags
  1899. * @param $target_organism_id
  1900. * @param $target_type
  1901. * @param $create_target
  1902. * @param $attr_locgroup
  1903. *
  1904. * @ingroup gff3_loader
  1905. */
  1906. private function loadTarget($feature, $tags, $target_organism_id, $target_type, $create_target, $attr_locgroup) {
  1907. // format is: "target_id start end [strand]", where strand is optional and may be "+" or "-"
  1908. $matched = preg_match('/^(.*?)\s+(\d+)\s+(\d+)(\s+[\+|\-])*$/', trim($tags['Target'][0]), $matches);
  1909. // the organism and type of the target may also be specified as an attribute. If so, then get that
  1910. // information
  1911. $gff_target_organism = array_key_exists('target_organism', $tags) ? $tags['target_organism'][0] : '';
  1912. $gff_target_type = array_key_exists('target_type', $tags) ? $tags['target_type'][0] : '';
  1913. // if we have matches and the Target is in the correct format then load the alignment
  1914. if ($matched) {
  1915. $target_feature = $matches[1];
  1916. $start = $matches[2];
  1917. $end = $matches[3];
  1918. // if we have an optional strand, convert it to a numeric value.
  1919. if ($matches[4]) {
  1920. if (preg_match('/^\+$/', trim($matches[4]))) {
  1921. $target_strand = 1;
  1922. }
  1923. elseif (preg_match('/^\-$/', trim($matches[4]))) {
  1924. $target_strand = -1;
  1925. }
  1926. else {
  1927. $target_strand = 0;
  1928. }
  1929. }
  1930. else {
  1931. $target_strand = 0;
  1932. }
  1933. $target_fmin = $start - 1;
  1934. $target_fmax = $end;
  1935. if ($end < $start) {
  1936. $target_fmin = $end - 1;
  1937. $target_fmax = $start;
  1938. }
  1939. // default the target organism to be the value passed into the function, but if the GFF
  1940. // file species the target organism then use that instead.
  1941. $t_organism_id = $target_organism_id;
  1942. if ($gff_target_organism) {
  1943. // get the genus and species
  1944. $success = preg_match('/^(.*?):(.*?)$/', $gff_target_organism, $matches);
  1945. if ($success) {
  1946. $values = array(
  1947. 'genus' => $matches[1],
  1948. 'species' => $matches[2],
  1949. );
  1950. $torganism = chado_select_record('organism', array('organism_id'), $values);
  1951. if (count($torganism) == 1) {
  1952. $t_organism_id = $torganism[0]->organism_id;
  1953. }
  1954. else {
  1955. $this->logMessage("Cannot find organism for target %target.",
  1956. array('%target' => $gff_target_organism), TRIPAL_WARNING);
  1957. $t_organism_id = '';
  1958. }
  1959. }
  1960. else {
  1961. $this->logMessage("The target_organism attribute is improperly formatted: %target. " .
  1962. "It should be target_organism=genus:species.",
  1963. array('%target' => $gff_target_organism), TRIPAL_WARNING);
  1964. $t_organism_id = '';
  1965. }
  1966. }
  1967. // default the target type to be the value passed into the function, but if the GFF file
  1968. // species the target type then use that instead
  1969. $t_type_id = '';
  1970. if ($target_type) {
  1971. $values = array(
  1972. 'name' => $target_type,
  1973. 'cv_id' => array(
  1974. 'name' => 'sequence',
  1975. )
  1976. );
  1977. $type = chado_select_record('cvterm', array('cvterm_id'), $values);
  1978. if (count($type) == 1) {
  1979. $t_type_id = $type[0]->cvterm_id;
  1980. }
  1981. else {
  1982. throw new Exception(t("The target type does not exist in the sequence ontology: %type. ",
  1983. array('%type' => $target_type)));
  1984. }
  1985. }
  1986. if ($gff_target_type) {
  1987. $values = array(
  1988. 'name' => $gff_target_type,
  1989. 'cv_id' => array(
  1990. 'name' => 'sequence',
  1991. )
  1992. );
  1993. // get the cvterm_id for the target type
  1994. $type = chado_select_record('cvterm', array('cvterm_id'), $values);
  1995. if (count($type) == 1) {
  1996. $t_type_id = $type[0]->cvterm_id;
  1997. }
  1998. else {
  1999. // check to see if this is a synonym
  2000. $sql = "
  2001. SELECT CVTS.cvterm_id
  2002. FROM {cvtermsynonym} CVTS
  2003. INNER JOIN {cvterm} CVT ON CVT.cvterm_id = CVTS.cvterm_id
  2004. INNER JOIN {cv} CV ON CV.cv_id = CVT.cv_id
  2005. WHERE CV.name = 'sequence' and CVTS.synonym = :synonym
  2006. ";
  2007. $synonym = chado_query($sql, array(':synonym' => $gff_target_type))->fetchObject();
  2008. if ($synonym) {
  2009. $t_type_id = $synonym->cvterm_id;
  2010. }
  2011. else {
  2012. $this->logMessage("The target_type attribute does not exist in the sequence ontology: %type.",
  2013. array('%type' => $gff_target_type), TRIPAL_WARNING);
  2014. $t_type_id = '';
  2015. }
  2016. }
  2017. }
  2018. // we want to add a featureloc record that uses the target feature as the srcfeature (landmark)
  2019. // and the landmark as the feature.
  2020. $this->loadFeatureLoc($feature, $organism, $target_feature, $target_fmin,
  2021. $target_fmax, $target_strand, $phase, $attr_fmin_partial, $attr_fmax_partial, $attr_residue_info,
  2022. $attr_locgroup, $t_type_id, $t_organism_id, $create_target, TRUE);
  2023. }
  2024. // the target attribute is not correctly formatted
  2025. else {
  2026. $this->logMessage("Could not add 'Target' alignment as it is improperly formatted: '%target'",
  2027. array('%target' => $tags['Target'][0]), TRIPAL_ERROR);
  2028. }
  2029. }
  2030. }