Merge branch 'rtd' into 7.x-3.x

.gitignore

@@ -5,3 +5,4 @@ tests/.env
 .buildpath
 .project
 .settings/
+docs/_build/

README.md

@@ -1,4 +1,6 @@
 [![7.x-3.x Build Status](https://travis-ci.org/tripal/tripal.svg?branch=7.x-3.x)](https://travis-ci.org/tripal/tripal)
+[![Documentation Status](https://readthedocs.org/projects/tripal/badge/?version=latest)](https://tripal.readthedocs.io/en/latest/?badge=latest)
+
 [![DOI](https://zenodo.org/badge/42666405.svg)](https://zenodo.org/badge/latestdoi/42666405)
 
 

docs/Makefile

@@ -0,0 +1,20 @@
+# Minimal makefile for Sphinx documentation
+#
+
+# You can set these variables from the command line.
+SPHINXOPTS    =
+SPHINXBUILD   = sphinx-build
+SPHINXPROJ    = Tripal
+SOURCEDIR     = .
+BUILDDIR      = _build
+
+# Put it first so that "make" without argument is like "make help".
+help:
+	@$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
+
+.PHONY: help Makefile
+
+# Catch-all target: route all unknown targets to Sphinx using the new
+# "make mode" option.  $(O) is meant as a shortcut for $(SPHINXOPTS).
+%: Makefile
+	@$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)

docs/_static/theme_overrides.css

@@ -0,0 +1,15 @@
+/* override table width restrictions
+See: https://rackerlabs.github.io/docs-rackspace/tools/rtd-tables.html
+ */
+@media screen and (min-width: 767px) {
+
+   .wy-table-responsive table td {
+      /* !important prevents the common CSS stylesheets from overriding
+         this as on RTD they are loaded after this stylesheet */
+      white-space: normal !important;
+   }
+
+   .wy-table-responsive {
+      overflow: visible !important;
+   }
+}

docs/conf.py

@@ -0,0 +1,160 @@
+# -*- coding: utf-8 -*-
+#
+# Configuration file for the Sphinx documentation builder.
+#
+# This file does only contain a selection of the most common options. For a
+# full list see the documentation:
+# http://www.sphinx-doc.org/en/master/config
+
+# -- Path setup --------------------------------------------------------------
+
+# If extensions (or modules to document with autodoc) are in another directory,
+# add these directories to sys.path here. If the directory is relative to the
+# documentation root, use os.path.abspath to make it absolute, like shown here.
+#
+# import os
+# import sys
+# sys.path.insert(0, os.path.abspath('.'))
+
+
+# -- Project information -----------------------------------------------------
+
+project = u'Tripal'
+author = u''
+
+# The short X.Y version
+version = u''
+# The full version, including alpha/beta/rc tags
+release = u'7.x-3.x'
+
+
+# -- General configuration ---------------------------------------------------
+
+# If your documentation needs a minimal Sphinx version, state it here.
+#
+# needs_sphinx = '1.0'
+
+# Add any Sphinx extension module names here, as strings. They can be
+# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
+# ones.
+extensions = [
+]
+
+# Add any paths that contain templates here, relative to this directory.
+templates_path = ['_templates']
+
+# The suffix(es) of source filenames.
+# You can specify multiple suffix as a list of string:
+#
+# source_suffix = ['.rst', '.md']
+source_suffix = '.rst'
+
+# The master toctree document.
+master_doc = 'index'
+
+# The language for content autogenerated by Sphinx. Refer to documentation
+# for a list of supported languages.
+#
+# This is also used if you do content translation via gettext catalogs.
+# Usually you set "language" from the command line for these cases.
+language = None
+
+# List of patterns, relative to source directory, that match files and
+# directories to ignore when looking for source files.
+# This pattern also affects html_static_path and html_extra_path .
+exclude_patterns = [u'_build', 'Thumbs.db', '.DS_Store']
+
+# The name of the Pygments (syntax highlighting) style to use.
+pygments_style = 'sphinx'
+
+
+# -- Options for HTML output -------------------------------------------------
+
+# The theme to use for HTML and HTML Help pages.  See the documentation for
+# a list of builtin themes.
+#
+html_theme = "sphinx_rtd_theme"
+
+# Theme options are theme-specific and customize the look and feel of a theme
+# further.  For a list of options available for each theme, see the
+# documentation.
+#
+# html_theme_options = {}
+
+# Add any paths that contain custom static files (such as style sheets) here,
+# relative to this directory. They are copied after the builtin static files,
+# so a file named "default.css" will overwrite the builtin "default.css".
+html_static_path = ['_static']
+
+html_context = {
+    'css_files': [
+        '_static/theme_overrides.css',  # override wide tables in RTD theme
+        ],
+     }
+
+# Custom sidebar templates, must be a dictionary that maps document names
+# to template names.
+#
+# The default sidebars (for documents that don't match any pattern) are
+# defined by theme itself.  Builtin themes are using these templates by
+# default: ``['localtoc.html', 'relations.html', 'sourcelink.html',
+# 'searchbox.html']``.
+#
+# html_sidebars = {}
+
+
+# -- Options for HTMLHelp output ---------------------------------------------
+
+# Output file base name for HTML help builder.
+htmlhelp_basename = 'Tripaldoc'
+
+
+# -- Options for LaTeX output ------------------------------------------------
+
+latex_elements = {
+    # The paper size ('letterpaper' or 'a4paper').
+    #
+    # 'papersize': 'letterpaper',
+
+    # The font size ('10pt', '11pt' or '12pt').
+    #
+    # 'pointsize': '10pt',
+
+    # Additional stuff for the LaTeX preamble.
+    #
+    # 'preamble': '',
+
+    # Latex figure (float) alignment
+    #
+    # 'figure_align': 'htbp',
+}
+
+# Grouping the document tree into LaTeX files. List of tuples
+# (source start file, target name, title,
+#  author, documentclass [howto, manual, or own class]).
+latex_documents = [
+    (master_doc, 'Tripal.tex', u'Tripal Documentation',
+     u'Stephen Ficklin, Lacey Sanderson, Bradford Condon et al', 'manual'),
+]
+
+
+# -- Options for manual page output ------------------------------------------
+
+# One entry per manual page. List of tuples
+# (source start file, name, description, authors, manual section).
+man_pages = [
+    (master_doc, 'tripal', u'Tripal Documentation',
+     [author], 1)
+]
+
+
+# -- Options for Texinfo output ----------------------------------------------
+
+# Grouping the document tree into Texinfo files. List of tuples
+# (source start file, target name, title, author,
+#  dir menu entry, description, category)
+texinfo_documents = [
+    (master_doc, 'Tripal', u'Tripal Documentation',
+     author, 'Tripal', 'One line description of project.',
+     'Miscellaneous'),
+]

docs/dev_guide.rst

@@ -0,0 +1,15 @@
+Developer's Guide
+==================
+
+
+.. toctree::
+   :maxdepth: 2
+   :caption: Table of Contents
+
+   dev_guide/introduction
+   dev_guide/data_structures
+   dev_guide/best_practices
+   dev_guide/custom_modules
+   dev_guide/custom_field
+   dev_guide/custom_data_loader
+   dev_guide/custom_web_services

docs/dev_guide/best_practices.rst

@@ -0,0 +1,64 @@
+Module Development Best Practice
+================================
+
+
+If you create custom Tripal Modules, here are some best practices and suggestions.
+
+The Drupal Devel Module
+=======================
+
+
+Before staring your development work, it is suggested that you download and install the `Drupal devel module <https://drupal.org/project/devel>`_. This module helps greatly with debugging your custom theme or module. A very useful function of this module is the dpm function. You can use the dpm function to print to the web page an interactive view of the contents of any variable. This can be extremely helpful when accessing Chado data in objects and arrays returned by Tripal.
+
+Add your module to Tripal.info
+==============================
+
+Do your best to list and upate your modules at http://tripal.info/extensions.
+
+
+Coding Best Practices
+=======================
+
+Host your code on GitHub
+-------------------------
+
+We recommend making your code open source and hosting it on GitHub. It’s free, it let’s people easily find, use, and contribute to your source code.
+
+Associate the GitHub repository with Tripal
+---------------------------------------------
+
+Once your module is on GitHub, consider joining the Tripal organization. Your lab group can exist as a team and maintain control over your code, but your projects will be listed in the main Tripal group.
+
+If you’d rather not, you can still tag your project as Tripal by clicking on the Manage Topics Link at the top of your repository.
+
+DOIs
+--------
+
+When your module is release ready, why not create a Digital Object Identifier (DOI) for it with `Zenodo <https://zenodo.org/>`_? It’s free! Sync your github account and create a new release (Zenodo won’t find old releases). You can then display your DOI badge on your module’s page.
+
+Additionally, there is a `Tripal Community group <https://zenodo.org/communities/tripal/>`_ on Zenodo. You can edit your record to associate your DOI with the Tripal community.
+
+Testing and Continuous Integration
+-----------------------------------
+
+`Tripal Test Suite <https://github.com/statonlab/TripalTestSuite>`_ is a full-featured testing module that makes writing tests much easier. which will automatically set up a PHPUnit and Travis testing environment for you.
+
+* Test with PHPUnit
+* Run tests as you push code with Travis CI
+
+
+Documentation
+--------------
+
+Every repository can include a README file that will be displayed on the repository page. A README file should at a minimum include:
+
+* An overview of the module
+* Instructions on how to install & use the module
+
+Consider documenting your Code itself. Tripal documents in the `Doxygen style <http://www.stack.nl/~dimitri/doxygen/>`_ which allows documentation webpages to be automatically generated. Even if you don’t build HTML documentation, the in-line code documentation will be very helpful to contributors.
+
+Coding Standards
+-----------------
+
+Drupal has defined `coding standards <https://www.drupal.org/docs/develop/standards/coding-standards>`_ that Tripal modules should meet.
+

docs/dev_guide/custom_data_loader.rst

@@ -0,0 +1,9 @@
+Creating Custom Data Loaders
+==============================
+
+
+.. note::
+
+  This documentation is currently being developed.
+
+

docs/dev_guide/custom_field.rst

@@ -0,0 +1,45 @@
+Creating a Custom Field
+=======================
+
+The most common way that new content will be added to an existing site is by creating new fields, or field displays.  In Tripal v2 customizations were added by editing PHP templates files.  These template files were  relatively easy to create and customize, but they provided less flexibility and did not integrate well with other Drupal features such as GUI-based page layout and Drupal Views.  Tripal v3 fields now provide this flexibility.  They also support data exchange and data collections!
+
+By default Tripal v3 provides many fields for display of Chado data. However, you may find that these fields do not display data as you want, or you want to display data that the current fields do not already provide. This section of the Handbook describes how to create new fields that are integrated into the display, search and exchange abilities of both Drupal and Tripal.
+
+If you are already familiar with Drupal fields you may be aware of the API functions and hooks that Drupal provides.  However, for the quantity of fields needed to support biological data, the Drupal API hooks quickly become overwhelming.  Additionally, documentation for fields in the Drupal API can someitmes be difficult to discover when first working with fields.   Therefore, Tripal provides several new PHP classes to simplify creation of fields and to consolidate all functionality into one easy to find set of files.  To develop new fields you should be somewhat familar working with PHP's Object-Oriented Classes. The new classes provided by Tripal are these:
+
+
++----------------------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
+| Class Name           | Description                                                                                                                                                                                                               |
++----------------------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
+| TripalField          | The TripalField class provides the basic information about a new field. It provides loaders for extracting data from the database and functions for querying data managed by the field.                                   |
++----------------------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
+| TripalFieldWidget    | The TripalFieldWidget class provides the necessary form elements when editing and Entity to allow the end-user to edit the value of the field (if desired). It provides the necessary validators and submittor functions. |
++----------------------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
+| TripalFieldFormatter | The TripalFieldFormatter class provides the visualization of the field when viewed on the page.                                                                                                                           |
++----------------------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
+| ChadoField           | The ChadoField class extends the TripalField class and provides the necessary settings to allow the field to map entities to data in Chado                                                                                |
++----------------------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
+| ChadoFieldWidget     | Extends the TripalFieldWidget class but currently provides no additional functionality. Use this class when working with Chado data to ensure future backwards compatibility.                                             |
++----------------------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
+| ChadoFieldFormatter  | Extends the TriplFieldFormatter class but currently provides no additional functionality. Use this class when working with Chado data to ensure future backwards compatibility.                                           |
++----------------------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
+
+
+
+The process for creating a custom field are as follows:
+
+* Determine the controlled vocabulary term that best describes the data your field will create.
+* Decide if you need the Chado field classes or the base Tripal field classes.  If you intend to work with data housed in Chado then you should use the Chado field classes.
+* Decide if you want to build your class manually from the ground up or speed development by using the Staton Lab Fields Generator tool.
+* Create new implementations of classes that extend those listed in the table above.  If you implement the functions properly your field is plug-and-play!  Tripal will find it and be able to use it.
+
+The rest of this section will walk you through these steps.
+
+
+.. toctree::
+   :maxdepth: 2
+   :caption: Table of Contents
+
+   custom_field/select_vocab_terms
+   custom_field/manual_field_creation
+   custom_field/tripal_field_generator

docs/dev_guide/custom_field/manual_field_creation.rst

@@ -0,0 +1,531 @@
+Manual Field Creation
+======================
+
+To show how a TripalField works we will break down a class implementation section by section.  Here we will use the **obi__organism** field that comes with Tripal and which extends the ChadoField class.  The ChadoField class is almost identical to the TripalField class except that it provides a few extra settings for working with Chado tables.   To create your own class you need to create a new class that implements the necessary functions.
+
+.. note::
+  Creation of your first field may not seem easy!  The following document is a lot to think about and consider. Therefore, when you write your first field, don't try to do everything at once. Take it one piece at a time.  The variables and functions described here are in order with the most critical components described first.  Take it at an even pace.
+
+
+
+Directory Structure for Fields
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+
+Before we create our class we must first create a proper directory structure.  Tripal expects that all new Tripal field classes are located inside of a custom module in the following directory structure:
+
+.. code-block:: bash
+
+  /sites/all/modules/[your_module]/includes/TripalField/[field_name]/[field_name].inc
+  /sites/all/modules/[your_module]/includes/TripalField/[field_name]/[field_name]_widget.inc
+  /sites/all/modules/[your_module]/includes/TripalField/[field_name]/[field_name]_formatter.inc
+
+
+In the directories above the token [your_module] can be substituted with the name of your module and [field_name] is the name of your field.  You can name your field whatever you like, but you must use this name consistently in other locations throughout the modules.  Because all fields are defined by vocabulary terms, it is custom to name your fields with the vocabulary **short name** followed by two underscores followed by the **term name**, hence:  obi__organism.  Here the ChadoField implementation goes in the [field_name].inc file, the ChadoFieldWidget in the [field_name]_widget.inc file and the ChadoFieldFormatter in the [field_name]_formatter.inc.   All new fields must implement all three classes.   in the case of our obi__organism field the directory structure is as follows:
+
+.. code-block:: bash
+
+  /sites/all/modules/tripal/tripal_chado/includes/TripalField/obi__organism/obi__organism.inc
+  /sites/all/modules/tripal/tripal_chado/includes/TripalField/obi__organism/obi__organism_widget.inc
+  /sites/all/modules/tripal/tripal_chado/includes/TripalField/obi__organism/obi__organism_formatter.inc
+
+Anatomy of the ChadoField Class
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+The following describes a ChadoField class from top to bottom. The code for the obi__organism field is shown in order that it appears in the class with descriptions provided for the meaning of each piece of code.  To write your own class, duplicate the variables and function and customize accordingly.  First, let's look at the definition of the class.  The following line defines the class and indicates that it extends the ChadoField class:
+
+.. code-block:: php
+
+  <?php
+
+  class obi__organism extends ChadoField {
+
+.. note::
+
+  In the line above, the class is named obi__organism. This must be the same name as the directory in which the field is located. Otherwise, Tripal won't be able to find the field.
+
+Static Member Variables
+~~~~~~~~~~~~~~~~~~~~~~~~
+
+Next, the TripalField/ChadoField class has a section of public static variables. These are variables that you can customize to describe your field to Tripal. Here you will provide the default label that appears for the field, and a description for the field:
+
+.. code-block:: php
+
+  <?php
+
+  // The default label for this field.
+  public static $default_label = 'Organism';
+
+  // The default description for this field.
+  public static $description = 'The organism to which this resource is associated.';
+
+As described in the section titled Tripal Data Structures, fields that are attached to Bundles are "instances" of a field. Every field instance can be customized differently on each bundle.  The following section of code allows your field to provide custom settings.  Here we want to "hard-code" the term that defines this field using the $default_instance_settings variable:
+
+
+.. code-block:: php
+
+  <?php
+  // Provide a list of instance specific settings. These can be access within
+  // the instanceSettingsForm.  When the instanceSettingsForm is submitted
+  // then Drupal with automatically change these settings for the instance.
+  // It is recommended to put settings at the instance level whenever possible.
+  // If you override this variable in a child class be sure to replicate the
+  // term_name, term_vocab, term_accession and term_fixed keys as these are
+  // required for all TripalFields.
+  public static $default_instance_settings  = array(
+    // The short name for the vocabulary (e.g. shcema, SO, GO, PATO, etc.).
+    'term_vocabulary' => 'OBI',
+    // The name of the term.
+    'term_name' => 'organism',
+    // The unique ID (i.e. accession) of the term.
+    'term_accession' => '0100026',
+    // Set to TRUE if the site admin is allowed to change the term
+    // type. This will create form elements when editing the field instance
+    // to allow the site admin to change the term settings above.
+    'term_fixed' => FALSE,
+    // The format for display of the organism.
+    'field_display_string' => '<i>[organism.genus] [organism.species]</i>',
+  );
+
+Notice in the code above that the elements **term_vocabulary, term_name** and **term_accession** are used to define the vocabulary term that this field maps to.  The term_fixed element allows the term to be changed by the site admin if desired.  These elements are required of all TripalFields classes.  You must always have these elements.  However, the **field_display_string** is a variable unique to this obi__organism field!  Because this field is displaying the organism we want to allow the site-admin to customize how the organism name is constructed and displayed.  Therefore, the **field_display_string** creates this new setting for us.  How this setting is used will be described later.
+
+As you may have noticed, a field requires a widget and a formatter.  This is why there are three classes for every field.  However, Drupal is flexible and allows fields to be edited or displayed by any number of widgets and formatters.  By default, Tripal provides one widget class and one formatter class for every field.  When you write a new field you will need to do the same and create a new ChadoFieldWidget and ChadoFieldFormatter class (or the corresponding non-Chado versions if you don't need Chado).  The following variables in the class indicate what are the default widget and formatter classes (we have not yet created those, but we know their names!):
+
+.. code-block:: php
+
+  <?php
+  // The default widget for this field.
+  public static $default_widget = 'obi__organism_widget';
+
+  // The default formatter for this field.
+  public static $default_formatter = 'obi__organism_formatter';
+
+Drupal allows new instances of fields to be attached to any Bundle.  This is really useful for fields like the built in Image field that Drupal provides.  It can be very handy to attache an instance of an Image field to any content type and viola! your content type now supports images.  However, there are some fields that should never be added via the online Drupal interface.  Our organism field is a good example.  We probably don't want to allow end-users to add an organism field to a Person content type...  In this case we will programmatically control which fields are attached to which Bundles.  We'll show that later.  But for now, let's set the no_ui variable to TRUE to prevent users from adding our new field to any Bundle.
+
+.. code-block:: php
+
+  <?php
+  // A boolean specifying that users should not be allowed to create
+  // fields and instances of this field type through the UI. Such
+  // fields can only be created programmatically with field_create_field()
+  // and field_create_instance().
+  public static $no_ui = TRUE;
+
+
+Sometimes a field is meant to provide a visualization or some other functionality.  An example of this might be a small search form or link to an analytical service.  In these cases we want the field to show up on the web page but it should not appear anywhere else, such as in Tripal's web service that provides access to all content.   We can set the no_data variable to TRUE and this will allow it to be seen on the site, but not anywhere else.
+
+.. code-block:: php
+
+  <?php
+  // A boolean specifying that the field will not contain any data. This
+  // should exclude the field from web serivces or downloads.  An example
+  // could be a quick search field that appears on the page that redirects
+  // the user but otherwise provides no data.
+
+
+  public static $no_data = FALSE;
+
+
+  .. note::
+  Be sure to only set this to TRUE when you are absolutely certain the contents would not be needed in web services.  Tripal was designed so that what appears on the page will always appear in web services.  Aside form the formatting we see on the website, the content should be the same.
+
+Finally, the last item in our Class variables is the **download_formatters**.  Tripal provides an API that allows tools to group entities into data collections.  Data collections are like "baskets" or "shopping carts".   Entities that are in data collections can be downloaded into files.  If your field is compatible with specific file downloaders you can specify those here.  A file downloader is a special TripalFieldDownloader class that "speaks" certain file formats.  Tripal, by default, provides the TripalTabDownloader (for tab-delimited files), the TripalCSVDownloader (for CSV files), a TripalNucFASTADownloader for creating nucleotide FASTA files and a TripalProteinFASTADownloader for protein FASTA files.   If your field is compatible with any of these formatters you can specify them in the following array:
+
+.. .. code-block::
+
+  <?php
+
+  // Indicates the download formats for this field.  The list must be the
+  // name of a child class of the TripalFieldDownloader.
+  public static $download_formatters = array(
+    'TripalTabDownloader',
+    'TripalCSVDownloader',
+  );
+
+
+If your field is compatible with the TripalTabDownloader, for example, your field will be included as a column in a tab delimited file where each row represents contents for a given entity.
+
+The load() function.
+~~~~~~~~~~~~~~~~~~~~~
+
+The first function we want to implement in our class is the load() function.   This function is responsible for querying the database and populating the field value.  Data that is loaded into the field must be organized in two ways: 1) a value that is visible to the end-users, and 2) values that are visible to Chado for ensuing update/editing of the correct record in Chado when the field is edited.  Our obi__organism field is designed to be used for multiple Bundles therefore the code in our load() function must be able to support any Chado table that has a foreign key relationship with the organism table.
+
+To get started, the load() function receives a single argument. The entity object:
+
+.. code-block:: php
+
+
+  public function load($entity) {
+
+
+Because this is a ChadoField and the TripalChado module supports this field and maps entities to their "base" record on Chado, we get something extra... we get the record itself
+
+.. code-block:: php
+
+
+    $record = $entity->chado_record;
+
+Having the record helps tremendously.  Our **obi__organism** field is meant to be attached to genomic feature content types (e.g. genes, mRNA, etc.), germplasm, etc.  Therefore, the entity will be a record of one of those types. In the case of a genomic feature, these come from the **feature** table of Chado.  In the case of a germplam, these records come from the **stock** table of Chado.  Both of these records have an **organism_id** field which is a foreign key to the organism table where we find out details about the organism.
+
+Before we set the values for our field, we need a little bit more information.  Remember that all field instances have settings?   The Tripal Chado module also populates for us the name of the Chado table and the column that this field maps to.  Our obi__organism field can be used for multiple Bundles.  A gene bundle would map to the **feature** table of Chado and a germplasm Bundle would map to the **stock** table.  We need to know what table and column this field is mapping to:  We can get that from the instance object of the class and its settings:
+
+.. code-block:: php
+
+
+    $settings = $this->instance['settings'];
+    $field_table = $this->instance['settings']['chado_table'];
+    $field_column = $this->instance['settings']['chado_column'];
+
+Next, we want to get this field name and its type.  We obviously know our field name, it is obi__organism.  However, we can get the name programmatically as well.  Drupal maintains an "informational" array about our field.  Inside of that field array we can find lots of interesting information such as our field name and its type (Bundle).  We'll need this when we set our field value.  But rather than hard-code it, let's grab it programmatically from the field name.  It's best to grab it programmatically because there are cases where the field name could change:
+
+.. code-block:: php
+
+
+    $field_name = $this->field['field_name'];
+    $field_type = $this->field['type'];
+
+
+Now, let's plan how we want our values to appear in our field.  The organism record of Chado v1.3 has a genus, species, abbreviation, infraspecific name, infraspecific type, and a common name.  We want these values exposed to the end user.  But, wait... when we discussed fields in the Tripal Data Structures section we learned about a name field that provides names for entities.  That field only has one value: the name.  Our organism field has multiple values (i.e. genus, species, etc.).   A field can provide more than just one value but values have to be qualified.  We have to provide values in key/value pairs, and the keys must be controlled vocabulary terms.  We must use controlled vocabulary terms because we want our field to be searchable by other Tripal sites.  For example, the ontology term for the word 'genus' comes from the TAXRANK vocabulary.  Fortunately, almost every column of every table in Chado has been mapped to a controlled vocabulary term so we don't need to go hunting for terms.  We can use a Chado API function that Tripal provides for getting the ontology terms associated with every column table in Chado.  The following code shows these functions retrieving the ontology terms for our values from the organism table:
+
+.. code-block:: php
+
+
+    // Get the terms for each of the keys for the 'values' property.
+    $label_term = 'rdfs:label';
+    $genus_term = tripal_get_chado_semweb_term('organism', 'genus');
+    $species_term = tripal_get_chado_semweb_term('organism', 'species');
+    $infraspecific_name_term = tripal_get_chado_semweb_term('organism', 'infraspecific_name');
+    $infraspecific_type_term = tripal_get_chado_semweb_term('organism', 'type_id');
+
+
+Notice that for our organism fields we can easily get the ontology terms for them using the API function **tripal_get_chado_semweb_term**.  You will also notice a **label_term** variable.  Sometimes a user may want to see the full name of the organism and not pieces of it in various elements.  Therefore, we will provide a label in our list of values that will concatenate the full organism name.  This field is not in our organism table so we hard-code the term 'rdfs:lable' which is a term from the Resource Data Framework Schema vocabulary that defines a label.
+
+Next, let's initialize our field's value to be empty.  When setting a field value we must do so in the entity object that got passed into our load function.  The entity is an object and it stores values using the names of the fields.  The following code sets an empty record for our field:
+
+.. code-block:: php
+
+
+    // Set some defaults for the empty record.
+    $entity->{$field_name}['und'][0] = array(
+      'value' => array(),
+    );
+
+
+Notice that our field has some sub elements. The first is 'und'.  This element corresponds to the "language" of the text.  Drupal supports mulitple spoken languages and wants to know the language of text we provide.  For Tripal fields we always use 'und' meaning 'undefined'.   The next element is the delta index number.  Field have a cardinality, or in other words they can have multiple values.  For every value we add we increment that index, always starting at zero.  The last element is our 'value' element and it is here where we put our element. You may notice that our **delta** index is hard coded to 0.  This is because an entity can only always have one organism that it is associated with.  We will never have more than one.
+
+Now that we've got some preliminary values and we've initialized our value array we can start adding values!  Before we do though, let's double check that we have a record.  If we don't have a record for this entity, we can't get a value.
+
+.. code-block:: php
+
+
+    if ($record) {
+
+
+Now if we do have a record we need to get the value  The first step is to actually get our organism record.  For this we will find the record variable to be really handy. It already comes pre-populated with every Chado record that has a foreign-key relationship with our base record.  So, in the case of a gene, the record is stored in the feature table which has an organism_id column which is a foreign key to the organism table.  So, we know then that our record object has an organism_id property and we can get our organism from that. The only exception is the biomaterial table which uses a field named taxon_id:
+
+.. code-block:: php
+
+
+      if ($field_table == 'biomaterial') {
+        $organism = $record->taxon_id;
+      }
+      else {
+        $organism = $record->organism_id;
+      }
+
+We can easily get all of the values we need from this organism object.   We can now access the values for this organism using the Chado organism table column names (e.g. $organism->genus, $organism->species).
+
+.. code-block:: php
+
+  <?php
+
+      $label = tripal_replace_chado_tokens($string, $organism);
+      $entity->{$field_name}['und'][0]['value'] = array(
+        $label_term => $label,
+        $genus_term => $organism->genus,
+        $species_term => $organism->species,
+      );
+      // The infraspecific fields were introduced in Chado v1.3.
+      if (property_exists($organism, 'infraspecific_name')) {
+        $entity->{$field_name}['und'][0]['value'][$infraspecific_type_term] = NULL;
+        $entity->{$field_name}['und'][0]['value'][$infraspecific_name_term] = $organism->infraspecific_name;
+        if ($organism->type_id) {
+          $entity->{$field_name}['und'][0]['value'][$infraspecific_type_term] =  $organism->type_id->name;
+        }
+      }
+
+In the code above we are populating our value array and we're using the controlled vocabulary terms we retrieved earlier as the keys.
+
+Okay, so, we have our values set. However, remember,  our fields must support two types of values: 1) those for end users; and 2) those that allow us to save values in Chado if the field is edited.  If you look at our value array above you will recognize that the entity to which this field is loading data for is for a feature or stock or library, etc.  This field represents the organism for a record from one of those tables.  If someone wants to edit the entity and change the organism  then effectively we need to change the organism_id of that table.  But in our values array we don't have the organism_id we only have data about the organism.  How will Tripal know how to change the organism for an entity if edited?  To do help Tripal out, we have to create special key/value pair to add to our values.  These are values that are not meant to be seen by the end-user.  The organism_id is a good example of such a value.  To create these values we create a key with a special naming scheme: use "chado-" as a prefix, followed by the table name (e.g. feature), followed by two underscores and finally the column name (e.g. organism_id).   The following code shows the creation of this value name:
+
+.. code-block:: php
+
+
+    // Set the linker field appropriately.
+    if ($field_table == 'biomaterial') {
+      $linker_field = 'chado-biomaterial__taxon_id';
+    }
+    else {
+      $linker_field = 'chado-' . $field_table . '__organism_id';
+    }
+
+If our entity were of type "gene" then our **field_table** is feature.  Therefore, our **linker_field** variable would be **chado-feature__organism_id**.  Next, we need to add this to our value:
+
+.. code-block:: php
+
+
+      $entity->{$field_name}['und'][0][$linker_field] = $organism->organism_id;
+
+Notice, though, that we did not add this value inside the 'value' key like we did above for our end-user, such as the following:
+
+.. code-block:: php
+
+  <?php
+
+  $entity->{$field_name}['und'][0]['value'])
+
+Instead, we put it in at the same level as 'value':
+
+.. code-block:: php
+
+  <?php
+
+  $entity->{$field_name}['und'][0][$linker_field]
+
+We do this because anything in the 'value' element is intended for the end-user.  Anything outside of the 'value' is meant for Tripal.  Adding the organism ID to this field as a Tripal "hidden" value allows Tripal to recognize where these values really came from.   When writing your own fields, you must include any values as "hidden" Tripal values that need to be written to the database table.  A good way to remember if you a value should be visible to the end-user or hidden for Tripal is to ask yourself these questions:
+
+  1.  Does the user need this value?  If yes, put it in the 'value' element.
+  2.  Does Tripal need the value when writing back to the Chado table?  If yes, put it as a hidden element.
+  3.  Does the user need to see the value an will this same value need to be written to the table?  If yes, then you have to put the value in both places.
+
+For our **obi__organism** field it is for entities with records in the **feature, stock, library**, etc. tables. Those tables only have an **organism_id** to represent the organism.  So, that's the database column this field is supporting.  We therefore, need to put that field as a hidden field, and all the others are just helpful to the user and don't get saved in the feature, stock or library tables. So, those go in the values array.
+
+Now, we're at a good stopping point with our field! We can close out our if($record) statement and the function:
+
+.. code-block:: php
+
+
+      }
+   }
+
+elementInfo() function
+~~~~~~~~~~~~~~~~~~~~~~
+The elementInfo() function is necessary to integrate your new field with Drupal Views and Tripal Web Services.  Drupal needs to know what data elements your field provides and Tripal needs to know what vocabulary terms to use for each of the data elements.  Related to vocabulary terms, all fields are assigned an ontology term for the field itself.  Every field has to have an one.   But when a field provides more than just a single data value it must also provide vocabulary terms for any sub elements as well.  Our obi__organism field provides the genus, species, etc. sub elements and, therefore, we need to describe these to Drupal and Tripal.  The elementInfo() function from the obi_organism field is as follows:
+
+.. code-block:: php
+
+  <?php
+
+  /**
+   * @see TripalField::elementInfo()
+   */
+  public function elementInfo() {
+    $field_term = $this->getFieldTermID();
+
+    $genus_term = chado_get_semweb_term('organism', 'genus');
+    $species_term = chado_get_semweb_term('organism', 'species');
+    $infraspecific_name_term = chado_get_semweb_term('organism', 'infraspecific_name');
+    $infraspecific_type_term = chado_get_semweb_term('organism', 'type_id');
+
+    return array(
+      $field_term => array(
+        'operations' => array('eq', 'contains', 'starts'),
+        'sortable' => TRUE,
+        'searchable' => TRUE,
+        'readonly' => FALSE,
+        'type' => 'xs:complexType',
+        'elements' => array(
+          'rdfs:label' => array(
+            'searchable' => TRUE,
+            'name' => 'scientific_name',
+            'operations' => array('eq', 'ne', 'contains', 'starts'),
+            'sortable' => FALSE,
+            'type' => 'xs:string',
+            'readonly' => TRUE,
+            'required' => FALSE,
+          ),
+          $genus_term => array(
+            'searchable' => TRUE,
+            'name' => 'genus',
+            'operations' => array('eq', 'ne', 'contains', 'starts'),
+            'sortable' => TRUE,
+            'readonly' => FALSE,
+            'type' => 'xs:string',
+            'required' => TRUE,
+          ),
+          $species_term => array(
+            'searchable' => TRUE,
+            'name' => 'species',
+            'operations' => array('eq', 'ne', 'contains', 'starts'),
+            'sortable' => TRUE,
+            'readonly' => FALSE,
+            'type' => 'xs:string',
+            'required' => TRUE,
+          ),
+          $infraspecific_name_term => array(
+            'searchable' => TRUE,
+            'name' => 'infraspecies',
+            'operations' => array('eq', 'ne', 'contains', 'starts'),
+            'sortable' => TRUE,
+            'readonly' => FALSE,
+            'type' => 'xs:string',
+            'required' => FALSE,
+          ),
+          $infraspecific_type_term => array(
+            'searchable' => TRUE,
+            'name' => 'infraspecific_type',
+            'operations' => array('eq', 'ne', 'contains', 'starts'),
+            'sortable' => TRUE,
+            'readonly' => FALSE,
+            'type' => 'xs:integer',
+            'required' => FALSE,
+          ),
+          'entity' => array(
+            'searchable' => FALSE,
+          ),
+        ),
+      ),
+    );
+  }
+
+
+The code above generates and returns an associative array that provides metadata about the field and its elements.  The array is structured such that the first-level key is the term for the field.  Details about the field are at the second-level and all sub elements are contained in a 'elements' key.  In the following code the terms for the field and sub elements are retrieved using TripalField class functions and Tripal API calls:
+
+.. code-block:: php
+
+  <?php
+
+    $field_term = $this->getFieldTermID();
+
+    $genus_term = chado_get_semweb_term('organism', 'genus');
+    $species_term = chado_get_semweb_term('organism', 'species');
+    $infraspecific_name_term = chado_get_semweb_term('organism', 'infraspecific_name');
+    $infraspecific_type_term = chado_get_semweb_term('organism', 'type_id');
+
+    return array( $field_term => array(
+      'operations' => array('eq', 'contains', 'starts'),
+      'sortable' => TRUE,
+      'searchable' => TRUE,
+      'readonly' => FALSE,
+      'type' => 'xs:complexType',
+      'elements' => array(
+
+Notice the value for $field_term variable was easily obtained by calling the $this->getFieldTermID function and all of the terms for the elements were obtained using the chado_get_semweb_term function which maps table columns in the Chado database schema to ontology terms.  The operations key indicates which search filter operations are supported for the field as a whole.  For this example these include 'eq' (for equals), 'contains' and 'starts' (for starts with).   The field is sortable and searchable so those values are set to TRUE.   Later, we weill learn how to implement the sorting, searching and filtering that the field will support.  For now we know we want them so we set the values accordingly.  Additionally, the field allows updating so 'readonly' is set to FALSE.   By convention, the 'type' of a field follows the XML data types for simple types (https://www.w3schools.com/xml/schema_simple.asp) and Complex types (https://www.w3schools.com/xml/schema_complex.asp) that have multiple elements.  Because our obi__organism field has subelements  and is not a single value, the field type is 'xs:complexType'.
+
+The array keys just mentioned fully describe our field to Drupal and Tripal.  Next we will define the sub elements in the same way, and these go in the 'elements' key.  First, we will describe the label.  Our obi__oranganism field provides a handly label element that concatenates the genus, species and infraspecific name into one simple string.  Therefore, we need to describe this element in the same way we described the field itself.  In the code below that the key is set to 'rdfs:label' (which is the controlled vocabulary term for a label) and that the child keys are the same as for the field.
+
+.. code-block:: php
+
+  <?php
+
+
+        'elements' => array(
+          'rdfs:label' => array(
+            'searchable' => TRUE,
+            'name' => 'scientific_name',
+            'operations' => array('eq', 'ne', 'contains', 'starts'),
+            'sortable' => FALSE,
+            'type' => 'xs:string',
+            'readonly' => TRUE,
+            'required' => FALSE,
+          ),
+
+Notice that our field will allow searching, provides a variety of search filter options, is sortable and defines the type as 'xs:string'.  The remaing elements follow the same pattern.  As another example, here is the genus element:
+
+.. code-block:: php
+
+  <?php
+
+         $genus_term => array(
+            'searchable' => TRUE,
+            'name' => 'genus',
+            'operations' => array('eq', 'ne', 'contains', 'starts'),
+            'sortable' => TRUE,
+            'readonly' => FALSE,
+            'type' => 'xs:string',
+            'required' => TRUE,
+          ),
+
+The major difference in the code above is that the term is provided by the variable $genus_term.
+
+Finally, our obi__organism field provides an 'entity' element that provides information for a published organism entity.  We do not provide any filtering, searching or sorting of those values.  So the final element appears as:
+
+.. code-block:: php
+
+          'entity' => array(
+            'searchable' => FALSE,
+          ),
+
+In summary,  you will always want to describe your field and every element of your field in the array returned by the elementInfo function.  However, you do not need to provide sorting, filtering or querying for every element.  If your field is read-only and simply provides values you should still describe these elements but you would set the meta data keys appropriately for the behavior of your field.   Also, you only need to describe elements in the values array returned by your load function.  Remember, there may be other key/value pairs (such as those used to help coordinate inserts/updates into Chado) but those do not need to be described here because they are never seen by the end-user.
+
+query() function
+~~~~~~~~~~~~~~~~~~
+
+
+As described above in the elementInfo function section, some fields and elements of fields are searchable.  if the elementInfo array indicates that the field is searchable and has operations (i.e. filters) then we must provide a way for those queries to occur.  This is where the query() function is needed.  The following is example code from the query function of our obi__organism field:
+
+.. code-block:: php
+
+  <?php
+
+  public function query($query, $condition) {
+      $alias = $this->field['field_name'];
+      $operator = $condition['operator'];
+
+      $field_term_id = $this->getFieldTermID();
+      $genus_term = $field_term_id . ',' . chado_get_semweb_term('organism', 'genus');
+      $species_term = $field_term_id . ',' . chado_get_semweb_term('organism', 'species');
+      $infraspecific_name_term = $field_term_id . ',' . chado_get_semweb_term('organism', 'infraspecific_name');
+      $infraspecific_type_term = $field_term_id . ',' . chado_get_semweb_term('organism', 'type_id');
+
+      // Join to the organism table for this field.
+      $this->queryJoinOnce($query, 'organism', $alias, "base.organism_id = $alias.organism_id");
+
+      // If the column is the field name then we're during a search on the full
+      // scientific name.
+      if ($condition['column'] == $field_term_id or
+          $condition['column'] == $field_term_id . ',rdfs:label') {
+        if (chado_get_version() <= 1.3) {
+          $query->where("CONCAT($alias.genus, ' ', $alias.species) $operator :full_name",  array(':full_name' => $condition['value']));
+        }
+        else {
+          $this->queryJoinOnce($query, 'cvterm', $alias . '_cvterm', 'base.infraspecific_type = ' . $alias . '_cvterm.type_id', 'LEFT OUTER');
+          $query->where("CONCAT($alias.genus, ' ', $alias.species, ' ', " . $alias . "'_cvterm.name', ' ', $alias.infraspecific_name) $operator :full_name",  array(':full_name' => $condition['value']));
+        }
+      }
+
+      // If the column is a subfield.
+      if ($condition['column'] == $species_term) {
+        $query->condition("$alias.species", $condition['value'], $operator);
+      }
+      if ($condition['column'] == $genus_term) {
+        $query->condition("$alias.genus", $condition['value'], $operator);
+      }
+
+      if ($condition['column'] == $infraspecific_name_term) {
+        $query->condition("$alias.infraspecific_name", $condition['value'], $operator);
+      }
+
+      if ($condition['column'] == $infraspecific_type_term) {
+        $this->queryJoinOnce($query, 'cvterm', 'CVT', "base.type_id = CVT.cvterm_id");
+        $query->condition("CVT.name", $condition['value'], $operator);
+      }
+    }
+
+The code above is how the field tells Drupal and Tripal how to find and filter the records that this field corresponds to.  First, we retreive the field alias and operators:and as with the load and elementInfo functions we get the controlled vocabulary terms for our field and field elements:
+
+
+.. code-block:: php
+
+  <?php
+
+    $alias = $this->field['field_name'];
+    $operator = $condition['operator'];
+
+    $field_term_id = $this->getFieldTermID();
+    $genus_term = $field_term_id . ',' . chado_get_semweb_term('organism', 'genus');
+    $species_term = $field_term_id . ',' . chado_get_semweb_term('organism', 'species');
+    $infraspecific_name_term = $field_term_id . ',' . chado_get_semweb_term('organism', 'infraspecific_name');
+    $infraspecific_type_term = $field_term_id . ',' . chado_get_semweb_term('organism', 'type_id');
+
+Next, our knowledge of Chado is needed.  We know that our obi__organism field will load data from the organism table.  Therefore, our search must occur there.

docs/dev_guide/custom_field/select_vocab_terms.rst

@@ -0,0 +1,157 @@
+Selecting Vocabulary Terms
+==========================
+
+Ontologies: what and why?
+-------------------------
+
+Tripal 3 requires all bundles and fields to be associated with a Controlled Vocabulary (CV). CVs are dictionaries of defined terms (CV terms) that make data machine-accessible, ensuring uniform terms are used across experiments, organisms and websites. Without CVterms, our scientific knowledge might be split by "dialects". Plant biologists might study temperature stress, while animal biologists study heat shock. Each group might benefit from the knowledge of the other, but they use a different vocabulary to describe the same thing, creating challenges for data discovery and exchange. CV terms make this easier not just for people, but especially for machines. Ontologies take this a step further. Where CVs are controlled lists of CVterms, ontologies are a controlled language, that include heirarchical relationships of terms.
+
+Tripal leverages vocabularies to make use of the `Semantic Web <https://en.wikipedia.org/wiki/Semantic_Web>`_. Every bundle and field defined in Tripal will be associated with a CVterm. Therefore, it is important to find community developed terms.  The `EMBL EBI Ontology Lookup Service <http://www.ebi.ac.uk/ols/index>`_ provides an easy location to search for and identify terms.  When choosing terms for new Bundles and Fields, think carefully about the terms you will use to describe your objects. Selecting the proper CV term that best describes the data may be the most challenging part of creating custom Bundles and Fields!
+
+Before you can create a new Bundle or Field  the vocabulary term must be present in your local Tripal site.  You can check if a term exists by using the Tripal lookup service on your local site using the URL path cv/lookup (e.g. http://your-site/cv/lookup).  If the term is not present then you'll need to add it.  You can do so manually by using Tripal's controlled vocabulary admin pages.  For creating new bundles this is all you need to do.  However, when creating Fields you will want to programmatically add the term.  This is important because Fields are meant to be shared. If you create an awesome field that you want to share with others then you need to make sure the terms get added programmatically.   The following sections describe how terms are stored in Chado and how you can add them using Tripal API calls.
+
+Storage of Terms in Chado
+-------------------------
+
+In Chado, CVs are stored by two tables: the **db** and **cv** tables. Chado was designed to store a record for the online database that a vocabulary lives at in the **db** table, and the namespace of a vocabulary in the **cv** table.  For example, the sequence ontology uses the namespace, sequence, which is stored in the **cv** table but uses the short name of SO which is stored in the **db** table.  As we'll see later, sometimes the distinction between what gets stored in the **cv** vs the **db** tables can get a bit fuzzy with some vocabularies. The terms themselves are stored in the cvterm table. The cvterm table has a foreign key to the **cv** table via the **cv_id** field.  Every controlled vocabulary term has an accession. For example the term gene in the Sequence Ontology has an accession number of SO:0000704.  Accession numbers consist of two parts: a vocabulary "short name", followed by a unique identifier separated by a colon.  Within Chado, the accession for any term is stored in the dbxref table.  This table has a foreign key to the **db** table via the **db_id** as well as a foreign key to the cvterm table via the **dbxref_id** field.
+
+Vocabulary Short Names and Namespaces
+-------------------------------------
+
+How can you tell what the **short name** and **namespace** values will be for a vocabulary term that you want to insert into Chado for your custom Bundle or Field? Hint: use the information is in the EMBL-EBI `Ontology Lookup Service <http://www.ebi.ac.uk/ols/index>`_ (OLS).  The following sections provide three examples for different cases.
+
+Case 1:  Ontologies without a defined namespace
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Consider the term for `organism <http://www.ebi.ac.uk/ols/ontologies/obi/terms?iri=http%3A%2F%2Fpurl.obolibrary.org%2Fobo%2FOBI_0100026>`_.
+
+.. figure:: select_vocab_terms.1.organism.png
+
+Notice how the teal box (the **short name**) is OBI, and the orange box contains the **full accession**, OBI:0100026 which includes but the **short name** and the unique term **accession** value.  Unfortunately, the OLS does not indicate the **namespace** terms.   So, as a rule we will use the short name converted to lower case.  Before using this term in a Tripal Bundle or Field you may need to insert this term into Chado.  You can do so in your custom module cude using the **tripal_insert_cvterm** function. The following provides a demonstration:
+
+.. code-block:: php
+
+	<?php
+
+   $term= tripal_insert_cvterm([
+            'id' => 'OBI:0100026',
+            'name' => 'organism',
+            'cv_name' => 'OBI',
+            'definition' => 'A material entity that is an individual living system, such as animal,
+            plant, bacteria or virus, that is capable of replicating or reproducing, growth and maintenance
+                  in the right environment. An organism may be unicellular or made up, like humans, of many
+               billions of cells divided into specialized tissues and organs.',
+        ]);
+
+
+
+Note that in the code above the namespace is provided as the **cv_name** element and the full accessions (including the short name) is provided as the **id** element.  In this case the OBI CV already exists by default in the Tripal database, so we did not need to add the vocabulary record.  If the OBI did not exist we could have added it using the following API calls.  First we insert the "database" record for the ontology.
+
+.. code-block:: php
+
+  <?php
+  tripal_insert_db(array(
+    'name' => 'obi',
+    'description' => 'The Ontology for Biomedical Investigation.',
+    'url' => 'http://obi-ontology.org/page/Main_Page',
+    'urlprefix' => 'http://purl.obolibrary.org/obo/{db}_{accession}',
+  ));
+
+
+
+Notice here that the **name** element is the **namespace** (short name converted to lower case) for the vocabulary.  The url is the web address for the ontology online. The urlprefix is a URL that can be used to construct a link that when clicked will take the user to any term in the vocabulary.  Almost all vocabularies will have a common URL for all terms.  Tripal will automatically substitute the short name into the **{db}** token and the term **accession** in to the **{accession}** token to generate the URL.
+
+Second, we insert the record for the controlled vocabulary.
+​
+
+.. code-block:: php
+
+  <?php
+ 	tripal_insert_cv(
+    	'OBI',
+    	'Ontology for Biomedical Investigation. The Ontology for Biomedical Investigations (OBI) is build in a collaborative, international effort and will serve as a resource for annotating biomedical investigations, including the study design, protocols and instrumentation used, the data generated and the types of analysis performed on the data. This ontology arose from the Functional Genomics Investigation Ontology (FuGO) and will contain both terms that are common to all biomedical investigations, including functional genomics investigations and those that are more domain specific.'
+  );
+
+
+Case 2:  Ontologies with a defined namespace
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Consider the entry for `CDS <https://www.ebi.ac.uk/ols/ontologies/so/terms?iri=http%3A%2F%2Fpurl.obolibrary.org%2Fobo%2FSO_0000316>`_.
+
+.. figure:: select_vocab_terms.2.cds.png
+
+
+Notice that in the Term Info box on the right there is the term **has_obo_namespace** which is defined as the word: sequence.  This is much better than the organism example from the OBI.  We now know the correct namespace for the term! By default, Tripal loads the Sequence Ontology during install.  However, suppose we did not have this term loaded we could do so with the following:
+
+
+.. code-block:: php
+
+  <?php
+    $term= tripal_insert_cvterm([
+            'id' => 'SO:0000316',
+            'name' => 'CDS',
+            'cv_name' => 'sequence',
+            'definition' => 'A contiguous sequence which begins with, and includes, a start codon and ends with, and includes, a stop codon. [ http://www.sequenceontology.org/browser/current_svn/term/SO:ma ].',
+        ]);
+
+Notice in the code above we can properly set the cv_name to sequence.
+
+Case 3: Ontologies with multiple namespaces
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Some ontologies are b into sub-ontologies. This includes the Gene Ontology (GO).  Let's consider the example GO term `cell aggregation <http://www.ebi.ac.uk/ols/ontologies/go/terms?iri=http%3A%2F%2Fpurl.obolibrary.org%2Fobo%2FGO_0098743>`_. Looking at the EBI entry, the teal box is GO, the orange box is GO:0098743, and the has_obo_namespace is biological_process. However, the GO provides two other namespaces:  cellular_component and molecular_function.  Be sure to pay attention to these different namespaces if you ever need to manually insert a term.
+
+.. figure:: select_vocab_terms.3.go.png
+
+
+Case 4: Ontologies with muliptle short names
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+The EDAM ontology builds its term accessions using different short names instead of the ontology. Consider the EDAM term for `Sequence <http://www.ebi.ac.uk/ols/ontologies/edam/terms?iri=http%3A%2F%2Fedamontology.org%2Fdata_2044>`_. The teal box is EDAM, the orange box is data:2044, and there is no **namespace**.
+
+.. figure:: select_vocab_terms.4.edam.png
+
+
+For this case, the **namespace** is EDAM, the short name is **data**, and the accession is 2044.  Unfortunately, this breaks the paradigm that Chado expects. Typically the **short name** is the teal box (EDAM).  In order to force Chado to properly handle ontologies like this we are forced to reverse the short name and **namespace** values when creating our record:
+
+
+.. code-block:: php
+
+  <?php
+	$term= tripal_insert_cvterm([
+	  'id' => 'data:2044',
+	  'name' => 'sequence',
+	  'cv_name' => 'EDAM',
+	  'definition' => 'One or more molecular sequences, possibly with associated annotation.',
+	]);
+
+	tripal_insert_db(array(
+	    'name' => 'data',
+	    'description' => 'Bioinformatics operations, data types, formats, identifiers and topics.',
+	    'url' => 'http://edamontology.org/page',
+	    'urlprefix' => 'http://edamontology.org/{db}_{accession}',
+	));
+
+	tripal_insert_cv(
+	    'EDAM',
+	    'EDAM is an ontology of well established, familiar concepts that are prevalent within bioinformatics, including types of data and data identifiers, data formats, operations and topics. EDAM is a simple ontology - essentially a set of terms with synonyms and definitions - organised into an intuitive hierarchy for convenient use by curators, software developers and end-users. EDAM is suitable for large-scale semantic annotations and categorization of diverse bioinformatics resources. EDAM is also suitable for diverse application including for example within workbenches and workflow-management systems, software distributions, and resource registries.'
+	);
+
+
+
+Case 5: You really cant find a term!
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Sometimes a good CVterm just doesn't exist for what you want to describe. If you can't find a CV term, you can insert a term into the "local" CV. This is meant to be used as a last resort. In these cases, before you use a local term, consider contributing the term to an existing CV or ontology. Any terms that are invented for a local site may mean that the data exposed by your site cannot be discovered by other sites or tools.  In this case, the accession will not be numeric,  but is the same as the term name.
+
+.. code-block:: php
+
+	<?php
+	$term= tripal_insert_cvterm([
+	   'id' => 'local:shame_on_you',
+	   'name' => 'shame_on_you',
+	   'cv_name' => 'local',
+	   'definition' => 'You should really find a good CVterm.',
+	]);
+
+Notice in the above code the **short name** and **namespace** are both "local" as this is a local term on the site.

docs/dev_guide/custom_field/tripal_field_generator.rst

@@ -0,0 +1,9 @@
+Easier Field Creation: Tripal Field Generator
+==============================================
+
+
+The Staton Lab has created a `specialized tool <https://github.com/statonlab/fields_generator>`_ for automatically generating a ChadoField or TripalField and populating them with the basic controlled vocabulary and database information.  Instructions are available at https://github.com/statonlab/fields_generator.
+
+.. figure:: tripal_field_generator.TFG.png
+   :align: center
+   :scale: 50%

docs/dev_guide/custom_modules.rst

@@ -0,0 +1,11 @@
+Creating Custom Modules
+========================
+
+.. note::
+
+	This section is under construction
+
+
+The Tripal API, in conjunction with the Drupal API allow developers to create their own modules that can "plug-in" with Tripal. Developers may create their own modules if they desire new or different functionality. A custom module can also be desired to house a sites "customizations." The Tripal API provides an interface for module developers to interact with the Chado database as well as the other Tripal core functions such as jobs management, materialized views, in-house vocabularies, external database cross-references, properties, etc. An understanding of the Drupal API and Drupal module development is required.
+
+Basic instructions for creating custom Drupal module can be found here:  https://www.drupal.org/docs/7/creating-custom-modules.   Before you can create custom modules that interact with Tripal, you should review those instructions and have practiced creating at least a simple module.   Tripal follows Drupal module development rules and API.  

docs/dev_guide/custom_web_services.rst

@@ -0,0 +1,347 @@
+Creating Custom Web Services
+==============================
+
+
+Introduction
+-------------
+
+New in Tripal v3 are `RESTful web services <https://en.wikipedia.org/wiki/Representational_state_transfer>`_.  These web-services are designed to allow end-users to access data programmatically using any programming language of their choice.  Web services in Tripal v3 are provided by the **tripal_ws** module.  By default the module provides the "content" service.  This service provides access via a RESTful interface to all of the content published on a Tripal site.  It is meant to respond immediately as the site admin makes changes to published data and is expected to always provide access to the same data displayed on the site (nothing more and nothing less). 
+
+
+Tripal v3 has been redesigned from Tripal v2 to be fully organized around controlled vocabularies.  All data made available via Tripal is expected to be described by the terms of a controlled vocabulary or ontology.  For example, all content types in Tripal are assigned a controlled vocabulary term that describes what the content is.  Additionally, each field of data attached to a content type also is described using a controlled vocabulary term.  If a field provides more than just a single data value (i.e. it provides a list or nested structured array of data of key/value pairs) then each of the keys for those pairs must also be a controlled vocabulary term.  This requirement allows Tripal to fully describe the data it houses to any other Tripal site and any other script or service that can read the web services.  As long as the client application understands the vocabulary term it will understand the meaning of the data.  Using controlled vocabulary terms to describe all data allows a Tripal site to participate in the `Semantic Web <https://en.wikipedia.org/wiki/Semantic_Web>`_. 
+
+Finally, the Tripal RESTful services are meant to be discoverable.  In some cases, when a web services is designed, the only way to understand the structure of it and the operations that it provides are for a programmer to read online documentation for the service before she can write the client application.  However, to better support automatic data discovery without human intervention by a client the Tripal web services have been designed to be discoverable.  To this end, Tripal uses the  `Hyrda Core Vocabulary <https://www.hydra-cg.com/spec/latest/core/>`_ specification.  It fully describes all of the services, their operations, and the resulting value.  A client application that understands the Hydra language can therefore learn  to use the web service without human intervention.  However, in practice, its a good idea to continue to provide online documentation for humans.  And this User's Guide :doc:`provides those instructions </user_guide/web_services>` for the default Tripal content service.  
+
+This documentation provides instructions to construct your own custom web services and making that service available through Tripal's existing web service infrastructure.  This will enable your web service to be discoverable and to provide a consistent experience to end-users.  Before proceeding with the following instructions, please review the **Structure of a Web Service Response** section on the User's Guide :doc:`Web Services page </user_guide/web_services>`.
+
+Getting Started
+----------------
+Before creation of your new web services you must first consider the following in your design:
+
+1.  What functionality will your service provide.  This will dictate the URL paths that you will need and what operations (Create, Read, Update, Delete) your service will support.
+2.  What controlled vocabularies do you need to use to describe the data that your service may provide.
+3.  What human-readable label can you give to your service.
+4.  What one-word describes the type of service you are creating.
+5.  What human-readable description should you provide to users of your service.
+
+When you are ready to begin construction of your web service, you must first create a custom Drupal module.  The assumption is that you already know how to create new Drupal modules, or you have access to an existing one into which you can add your new web services.   Prepare your custom module by creating the following directory structure:
+
+  .. code-block:: bash
+
+	[module_name]/includes/TripalWebService
+
+Where [module_name] is the name of your custom module.  It is inside of this directory that you will place the code for your new web services.  Tripal is designed to recognize this directory, discover the web services described inside of it and to automatically make them available!  You need only program the service and Tripal does the rest.  
+
+
+**Note:** When selecting a version number it is best practice to start with 0.1.  The first number represents the  "major" number and the second number is the "minor" number.  When minor bug fixes or changes to the service occur the minor number should be incremented. When major changes occur that affect the structure of the web service or it's operations then the major number should be incremented.    The service can be named whatever you like.   This class file should be named according to this schema with a **.inc** extension. 
+
+For this tutorial, suppose we wanted to create a web service that allowed someone to interact with the Tripal Job queue.  We could name our new class the **TripalJobService_v0.1** class and we would create this class in the file: 
+
+.. code-block:: bash
+
+	[module_name]/includes/TripalWebService/TripalJobService_v0_1.inc
+
+Within this file we will implement our class with the following structure:
+
+
+.. code-block:: php
+
+	<?php
+	class TripalJobService_v0_1 extends TripalWebService {
+
+	  /**
+	   * The human-readable label for this web service.
+	   */
+	  public static $label = 'Jobs';
+
+	  /**
+	   * A bit of text to describe what this service provides.
+	   */
+	  public static $description = 'Provides interaction with the Tripal Job Queue';
+
+	  /**
+	   * A machine-readable type for this service. This name must be unique
+	   * among all Tripal web services and is used to form the URL to access
+	   * this service.
+	   */
+	  public static $type = 'jobs';
+
+	  /**
+	   * Implements the constructor
+	   */
+	  public function __construct($base_path) {
+	    parent::__construct($base_path);
+	  }
+	}
+
+
+This is all we need for Tripal to recognize our new service!  Notice that the class implementation extends the TripalWebSerivce class and it sets a few static variables that defines the label, description and name for this service.  Finally, this class defines the constructor which simply calls the parent class constructor. Be sure to always call the parent constructor when you implement your own service.  We can now use the `Hydra console <http://www.markus-lanthaler.com/hydra/console/>`_ to see our service.  Note, that the hydra console must be able to have access to your site. For this tutorial, the Tripal site is temporarily hosted on the local machine, and hence Hydra has been installed locally.    To see if your new service shows up, enter the URL for your site into the Hydra console and you should see it appear:
+
+.. image:: custom_web_services.1_hydra_console.png
+
+Notice in the above screen shot that our **jobs** service is now present in the **Response** section, and in the **Documentation** section!   The **Response** section shows the JSON array returned by Tripal and the **Documentation** section displays the information about our service.
+
+Documenting the Services
+-------------------------
+
+Our service appears in the Hydra console but if we try to use Hydra to perform a **GET** operation on our Jobs service there will be no interesting response and no documentation.  Try this by clicking on the link in the **Response** section for our Jobs service, and selecting the GET operation.
+
+.. image:: custom_web_services.2.png
+
+
+
+You will see that our service provides nothing:
+
+.. image:: custom_web_services.3.png
+
+
+Before we create our service we should have planned the design of our service. Suppose for now we just wanted to provide read-only access to job information.  Our design for the web service is quite simple and consists of the these resources (URLs and operations):
+
+
+TABLE!
+
+
+
+Before we begin implementation of our web service we must first document these resources.  To do this we must add a new function to our TripalJobService_v0_1 class named **getDocumentation**.  The following code shows this initial implementation of that function in our class:
+
+
+.. code-block:: php
+
+	 /**
+	     * @see TripalWebService::getDocumentation()
+	     */
+	    public function getDocumentation() {
+	        return parent::getDocumentation();
+	    }
+
+Notice currently all this function does is call the parent getDocumentation function.  Now, the first thing we need to document is our web service classes.  A web service class (not to be confused with the PHP class) simply refers to a resource.  A resource is any distinct URL within web services.  So, according to our design above we have two resources and hence two classes:  a jobs collection resource and a job resource.  Tripal must describe all of the classes (i.e. resources) using the Hydra method.  This makes the web service discoverable.   For example, with the content web service, Tripal provides a resource for each content type.  The Gene content type that Tripal provides is  described using the Hydra method in a JSON array with the following:
+
+
+.. code-block:: json
+
+	{
+	  "@id": "http://www.sequenceontology.org/browser/current_svn/term/SO:0000704",
+	  "@type": "hydra:Class",
+	  "hydra:title": "Gene",
+	  "hydra:description": "A region (or regions) that includes all of the sequence elements necessary to encode a functional transcript. A gene may include regulatory regions, transcribed regions and\/or other functional sequence regions. [SO:immuno_workshop]",
+	  "subClassOf": "hydra:Resource",
+	  "supportedOperation": [
+	    {
+	      "@id": "_:gene_retrieve",
+	      "@type": "hydra:Operation",
+	      "method": "GET",
+	      "label": "Retrieves the Gene resource.",
+	      "description": null,
+	      "statusCodes": [],
+	      "expects": null,
+	      "returns": "http://www.sequenceontology.org/browser/current_svn/term/SO:0000704"
+	     }
+	  ]
+	}
+
+In the above array, notice the @id is URL that represents a unique identifier for the class.   The @type will always be 'hydra:Class' because we are documenting a resource.  Then there is information about the class defined using the 'hydra:title' and 'hydra:description'.  The 'subclassOf' is always set to 'hydra:Resource'.  Next is the list of supported operations for this resource.  Remember, in our design we only want to support the GET operation for our Jobs service, so just like in the example above, the method we will support is GET.  The key/value pairs for the GET method are described using Hydra terms.  
+
+For our services we need to provide the information to Tripal so that it can generate these Hydra JSON arrays that document our service.  Tripal provides some easy API functions to help with this.  The firt is the **addDoc** member function.  This function will take as input the class details, operations and properties necessary to generate the documentation for our class.  First, lets use this function to document our Jobs Collection resource.  Below is sample code that will do this for us.
+
+
+.. code-block:: php
+
+	<?php
+
+ 	public function getDocumentation() {
+        $term = tripal_get_term_details('local', 'computational_jobs');
+        $details = array(
+            'id' => $term['url'],
+            'title' => $term['name'],
+            'description' => $term['definition'],
+        );
+        $operations = array(
+            'GET' => array(
+                'label' => 'Computational Jobs',
+                'description' => 'Retrieves the list of computational jobs that have been submitted on this site.',
+                'returns' => $term['url'],
+                'type' => '_:computational_jobs_retrieve',
+            ),
+        );
+        $properties = array(
+        );
+        $this->addDocClass($details, $operations, $properties);
+        return parent::getDocumentation();
+    
+
+
+In the code above we add the documentation for our Job Collection class. There are three different arrays, one for the class details, one for the operations that the class supports and the third for properties. For now, the properties array is left empty. We'll come back to that later.  All classes must use a controlled vocabulary term.  Notice that the term used for this class is a term local to the database named 'computational_jobs'.   Normally when creating a class we would try to use a term from a published controlled vocabulary.  A large number of these vocabularies can be searched using `the EBI Ontology Lookup Service <https://www.ebi.ac.uk/ols/index>`_.  Unfortunately, an appropriate term could not be found in a published vocabulary, so we had to create a local term.  We can use Tripal's API functions to easily add new terms.  The following code should be placed in the install() function of your module to ensure the term is available:
+
+
+.. code-block:: php
+
+    $term = tripal_insert_cvterm(array(
+            'id' => 'local:computational_job',
+            'name' => 'Computational Job',
+            'cv_name' => 'local',
+            'definition' => 'A computational job that executes a specific task.',
+        ));
+        $term = tripal_insert_cvterm(array(
+            'id' => 'local:computational_jobs',
+            'name' => 'Computational Jobs',
+            'cv_name' => 'local',
+            'definition' => 'A set of computational jobs where each job executes a specific task.',
+        ));
+
+You'll notice in the code above that the @id of the Class is the URL of the term.  Using the **tripal_get_term_details** function we can get the URL for the term.  The URL serves as the unique identifier for this term.  We simply set the title and description for the class using the term details.  For the operation, we can specify any of the HTTP protocols (e.g. GET, PUT, PUSH, DELETE and PATCH).  Here we are currently only supporting read-only operations so we only need to provide a 'GET' operation.    Our jobs collection resource is now documented!
+
+
+Implementing a Collection Resource
+--------------------------------------
+
+Now that our job collection resource is documented we can implement the resource using the **handleRequest** function.  We currently only support two paths for our web services as indicated in our design table above. Those include the default path where our job collection resource is found and an additional path with the job ID appended where individual job resources are found.  First, we will implement the Job Collections resource:
+
+
+.. code-block:: php
+
+    /**
+     * @see TripalWebService::handleRequest()
+     */
+    public function handleRequest() {
+
+        // Get the content type.
+        $job_id = (count($this->path) > 0) ? $this->path[0] : '';
+
+        // If we have a content type then list all of the entities that belong
+        // to it.
+        if (!$job_id) {
+            $this->doJobsList();
+        }
+    }
+
+in the code above need to determine if the resource is a job collection or a job resource.  To do that we can check to see if a job_id was provided.  The TripalWebService class provides as a member element the full URL path broken into an array of elements.  Because our job_id would always be in the first element of the path (after our base path for the service) we can use **$this->path[0]** to look for a job_id.  If one is not provided then we can execute a function called **doJobsList** and the code for that is as follows:
+
+
+.. code-block:: php
+
+	<?php
+
+    /**
+     * Generates the job collection resource.
+     */
+
+    private function doJobsList() {
+        // If the user has specified a limit or page number then use those to
+        // get the specific jobs.
+        $limit = isset($this->params['limit']) ? $this->params['limit'] : '25';
+        $page = isset($this->params['page']) ? $this->params['page'] : 0;
+
+        // Get the list of jobs for the given page, and the total number.
+        $offset = $page * $limit;
+        $jobs = tripal_get_jobs($offset, $limit);
+        $num_records = tripal_get_jobs_count();
+
+        // Set the current resource to be a new TripalWebServiceCollection resource,
+        // and pass in the current service path, and set the pager.
+        $service_path = $this->getServicePath();
+        $this->resource = new TripalWebServiceCollection($service_path, $this->params);
+        $this->resource->setType('local:computational_jobs');
+        $this->resource->initPager($num_records, $limit, $page);
+
+        // Now add the jobs as members
+        foreach ($jobs as $job) {
+            $member = new TripalWebServiceResource($service_path);
+            $member->setID($job->job_id);
+            $member->setType('local:computational_job');
+            $member->addProperty('schema:ItemPage', url('admin/tripal/tripal_jobs/view/' . $job->job_id, array('absolute' => TRUE)));
+            $this->resource->addMember($member);
+        }
+    }
+
+The first few lines of code above are as follows:  
+
+.. code-block:: php
+
+        $limit = isset($this->params['limit']) ? $this->params['limit'] : '25';
+        $page = isset($this->params['page']) ? $this->params['page'] : 0;
+
+
+
+Remember, that we wanted to allow for paging of our job collection.  We could have hundreds or thousands of jobs over time and we do not want to slow the page load by loading all of those jobs.  Therefore the page and limit parameters that can be added to the URL are available via the params member as a set of key/value pairs.  Next, using Tripal API function calls, we get the list of jobs that the user has requested to see (or the first page by default):
+
+.. code-block:: php
+
+      $offset = $page * $limit;
+        $jobs = tripal_get_jobs($offset, $limit);
+        $num_records = tripal_get_jobs_count();
+
+
+Now that we have our list of jobs to use in the collection we next need to build the resource.  We do this by setting the resource member of our TripalJobService_v0_1 class.  Tripal provides an easy way for constructing a collection via a class named TripalWebServiceCollection.  This class provides the necessary functions to easily create a collection resource that in the end will generate the appropriate JSON for us.  To create a collection we first instantiate a new instance of the TripalWebServiceCollection class and pass it the URL path that it corresponds to (in this case our base service path for the service).  We assign this new object to the resource member of our class.
+
+.. code-block:: php
+
+        $service_path = $this->getServicePath();
+        $this->resource = new TripalWebServiceCollection($service_path, $this->params);
+
+Next we need to indicate what type of collection this is.  Remember the controlled vocabulary terms we created previously?  We need to use those again to set the type.  Our term for a job collection is: local:computational_jobs.  So, we need to use this to set the type:
+
+.. code-block:: php
+
+        $this->resource->setType('local:computational_jobs');
+
+Now, because we have instantiated a TripalWebServiceCollection object it can handle creation of the pager for us. We just need to tell it how many total records there are, the page and number of records per page (i.e. limit):
+
+.. code-block:: php
+
+        $this->resource->initPager($num_records, $limit, $page);
+
+Lastly, we need to add our "members" of the collection.  These are the jobs from our query.  The following for loop iterates through alll of our jobs and creates new member objects that are added to our collection:
+
+.. code-block:: php
+
+	<?php
+
+        foreach ($jobs as $job) {
+            $member = new TripalWebServiceResource($service_path);
+            $member->setID($job->job_id);
+            $member->setType('local:computational_job');
+            $member->addProperty('schema:name', $job->job_name);
+            $member->addProperty('schema:ItemPage', url('admin/tripal/tripal_jobs/view/' . $job->job_id, array('absolute' => TRUE)));
+            $this->resource->addMember($member);
+        }
+
+Notice in the code above that each job is an instance of the class called TripalWebServiceResource.  We use this class because each element of the collection is a reference to a resource and we reference the ID and the type.   In the code above we create the new member resource, we set it's type to be the vocabulary term 'local:computational_job' and eventually, use the addMember function of our TripalWebServiceCollection to add it to the collection. 
+
+Also in the code above is a new function named addProperty.   We want to add some additional information about the job to help the end-user understand what each job is and how to get to it.  Here we add two properties, one that is the job name and another that is the page URL for the job on our Tripal site.  With these properties the client can quickly see the title and can go see the job on the Tripal site by following the given URL.  Note, a resource always has two identifying pieces of information: the ID and the Type.  So, everything else is added as a property of the resource.   Also notice that the first argument when using the addProperty function is a controlled vocabulary term.  Here we've used the terms **schema:name** and **schema:ItemPage**.  These terms are from the Schema.org vocabulary and the define what these properties are: a name and an item's page.
+
+Now that we have what looks like enough code to handle the job collection resource, we can return to Hydra to test if our new resource is working.  The screenshot below shows the results:
+
+.. image:: custom_web_services.4.png
+
+
+It's clear that our resource is working!  However, there are some issues.  The URL for the ItemPage does not show as clickable, and we're missing descriptions of the properties in the Documentation column on the right. We'll fix that issue a bit later.  For now, this looks good.
+
+
+
+Simplifying the Property Keys
+-----------------------------
+
+.. note::
+
+	The rest of the guide is under construction
+
+
+Implementing a Resource
+------------------------
+ 
+Documenting Properties
+------------------------
+
+More on Parameters
+------------------------
+
+Implementing Other Operations
+------------------------------
+
+Controlling Access to Resources
+---------------------------------
+
+
+ 
+
+Debugging and Troubleshooting

docs/dev_guide/data_structures.rst

@@ -0,0 +1,59 @@
+Tripal Data Structures
+=======================
+
+This page explains the relationships between Entity types, Bundles (content type), Entities and Fields. These are data structures provided by Drupal that Tripal v3 uses to expose biological content through your Drupal site, and to provide flexibility in site customization.  It is important to understand these terms and their relationships before developing new modules or custom functionlaity.   A quick summary of these Drupal data structures are as follows:
+
+* Entity:  a discrete data record.  Entities are most commonly are seen as "pages" on a Drupal web site.
+* Field:  an "atomic" piece of ancillary data that describes, defines or expands the entity.  Common fields include the name of an entity, a "body" of descriptive text, etc.
+* Bundle:  a content type.  An entity must always have a content type.  Drupal provides several content types by default:  Basic Page and Article.  Tripal provides biological bundles (e.g. genes, organisms, etc).
+* Entity Type:  despite the confusing name, an entity type is simply a group of bundles that are somehow related.  Drupal provides a "Node" Entity type that includes the Page and Article bundles.  All of the Tripal bundles (content types) belong to the TripalEntity Entity type.
+
+
+The following figure describes the heirarchical relationship between Drupal Entity types (e.g. Node) in comparison with TripalEntity entity types (e.g. Chromosome, Germplasm, etc.). 
+
+
+.. figure:: /_images/dev_guide/data_structures/Terminology-Diagram.png
+   :scale: 100 %
+   :alt: Entity terminology guide
+
+
+Furthermore, fields are "attached" to a Bundle and hold unique values for each Entity. The following figure describes this relationship for a Gene Bundle that has several fields attached: name, description and organism.  Note that in this figure the Entity and each of the Fields are defined using a controlled vocabulary term.  As a result, bundles and fields can be described using the `Semantic Web <https://en.wikipedia.org/wiki/Semantic_Web>`_ concepts of a "subject", "predicate" and "object".  We will discuss more on controlled vocabularies a bit later in the Handbook.
+
+
+
+.. figure:: /_images/dev_guide/data_structures/Tripal-Semantic-web.png
+   :scale: 50 %
+   :alt: Semantic web
+   
+
+
+Bundles (Content Types)
+=======================
+
+Bundles are types of content in a Drupal site.  By default, Drupal provides the Basic Page and Article content types, and Drupal allows a site developer to create new content types on-the-fly using the administrative interface--no programming required.  Tripal also provides several Content Type by default. During installation of Tripal the Organism, Gene, Project, Analysis and other content types are created automatically.  The site developer can then create new content types for different biological data--again, without any programming required.
+
+In order to to assist with data exchange and use of common data formats, Tripal Bundles are defined using a controlled vocabulary term (cvterm). For example, a "Gene" Bundle is defined using the Sequence Ontology term for gene whose term accession is: SO:0000704. This mapping allows Tripal to compare content across Tripal sites, and expose data to computational tools that understand these vocabularies. You can create as many Tripal Content Types as you would like through Administration > Structure > Tripal Content Types provided you can define it using a controlled vocabulary term. 
+
+By default, Tripal uses Chado as its primary data storage back-end.  When a bundle is created, the Tripal Chado module allows you to map a Bundle to a table in Chado.  Thus, any content type desired can be define as well as how it is stored in Chado--all using the administrative interface.  
+
+Entity
+======
+
+An entity is a discrete data record.  Entities are most commonly seen as "pages" on a Drupal web site and are instances of a Bundle (i.e content type). When data is published on a Tripal site such as organisms, genes, germplasm, maps, etc., each record is represented by a single entity with an entity ID as its only attribute.   All other information that the entity provides is made available via Fields.  
+
+Fields
+=======
+
+A field is a reusable "data container" that is attached to a Bundle. Programmatically, each field provides one or more primitive data types, with validators and widgets for editing and formatters for display. Each field independently manages the data to which it assigned.  Just like with Bundles, Fields are also described using controlled vocabulary terms.  For example, a gene Bundle has a field attached that provides the name of the gene.   This field only provides the name and nothing more.  Tripal uses the `schema:name <http://schema.org/name>`_ vocabulary term to describe the field.  
+
+Field Instances
+================
+
+Fields describe "atomic" units of data that are associated with an entity.  For example, a "name" is an atomic unit of data about a Gene or Organism entity. Fields can be reused for multiple Bundles. For example, gene, mRNA, genetic markers and variants all have name data.  Despite that all of these Bundles provides a "name", we only need one field to describe that this data is a "name".  However, we may want to customize a field specific to each bundle.  Therefore, an Instance of a field is attached to a bundle, and field instances can then be customized differently.  The most important customization is the one that defines the Chado table from which the data for a field is retrieved.   Despite that field instances are attached to bundles, they become visible with Entities.  When an entity is loaded for display, Drupal examines all of the fields that are attached to the entity's bundle, and then populates the fields instances with data specific to the entity being loaded.
+
+Entity Types
+=============
+
+An entity type is simply a group of Bundles that have some similarity.  For examples Drupal provides a Node entity type. The Node entity type contains the Basic Page and Article Bundles.  Tripal v2 expanded the Node entity type when creating new content.  Tripal v3, however, uses an a new entity type named TripalEntity that provides the Organism, Gene, Analysis, etc. content types.  Using these new entity types provides a a more responsive solution then the Node entity type, is more flexible, and supports the new ontology-driven approach of Tripal v3.
+
+ 

docs/dev_guide/introduction.rst

@@ -0,0 +1,31 @@
+
+Introduction to the Tripal API
+==============================
+
+Tripal provides an Application Programming Interfaces (API) that allows developers to interact with and customize Tripal. Using the API, developers can customize the way data is presented or create custom modules that provide new or different functionality. These custom modules can in turn be shared with the Tripal community. The Tripal API is documented online at http://api.tripal.info/api/tripal/3.x (COMING SOON). This document provides examples and best practices for using the Tripal API.
+
+Requirements
+-------------------------------
+
+In order to use the Tripal API the developer should have the following skills:
+
+Common skills:
+
+* Knowledge of PHP.
+* Knowledge of relational databases and SQL.
+
+To create custom modules:
+
+* Familiarity with Drupal API
+* Familiarity with Drupal module development
+
+To use Chado for data storage:
+
+* Knowledge of Chado and relationships between tables (at least tables where data of interest is stored).
+* An idea how data is stored in Chado (or a willingness to ask questions)
+
+Things that will make your Tripal development experience postive, fun and rewarding:
+
+* A desire to write code that can be re-used and shared by other groups
+* A desire to share your work with others to support other the Tripal community members.
+* A willingness to ask for help on the Tripal Github issue queue if you get stuck, find bugs or desire new features.

docs/index.rst

@@ -0,0 +1,14 @@
+.. Tripal documentation master file, created by
+   sphinx-quickstart on Tue Aug 21 17:25:20 2018.
+   You can adapt this file completely to your liking, but it should at least
+   contain the root `toctree` directive.
+
+Welcome to Tripal's documentation!
+==================================
+
+.. toctree::
+   :maxdepth: 4
+   :caption: Contents:
+
+   user_guide
+   dev_guide

docs/make.bat

@@ -0,0 +1,36 @@
+@ECHO OFF
+
+pushd %~dp0
+
+REM Command file for Sphinx documentation
+
+if "%SPHINXBUILD%" == "" (
+	set SPHINXBUILD=sphinx-build
+)
+set SOURCEDIR=.
+set BUILDDIR=_build
+set SPHINXPROJ=Tripal
+
+if "%1" == "" goto help
+
+%SPHINXBUILD% >NUL 2>NUL
+if errorlevel 9009 (
+	echo.
+	echo.The 'sphinx-build' command was not found. Make sure you have Sphinx
+	echo.installed, then set the SPHINXBUILD environment variable to point
+	echo.to the full path of the 'sphinx-build' executable. Alternatively you
+	echo.may add the Sphinx directory to PATH.
+	echo.
+	echo.If you don't have Sphinx installed, grab it from
+	echo.http://sphinx-doc.org/
+	exit /b 1
+)
+
+%SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS%
+goto end
+
+:help
+%SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS%
+
+:end
+popd

docs/user_guide.rst

@@ -0,0 +1,23 @@
+User's Guide
+==============
+
+
+.. toctree::
+   :maxdepth: 1
+   :caption: Table of Contents
+   :glob:
+
+   user_guide/what_is_tripal
+   user_guide/install_tripal
+   user_guide/drupal_overview
+   user_guide/example_genomics
+   user_guide/learn_chado
+   user_guide/creating_content
+   user_guide/setting_page_urls
+   user_guide/configuring_page_display
+   user_guide/mviews
+   user_guide/searching
+   user_guide/job_management
+   user_guide/web_services
+   user_guide/bulk_loader
+   user_guide/customize_site

docs/user_guide/bulk_loader.rst

@@ -0,0 +1,317 @@
+
+Bulk Loader
+===========
+
+The bulk loader is a tool that Tripal provides for loading of data contained in tab delimited files. Tripal supports loading of files in standard formats (e.g. ``FASTA``, ``GFF``, ``OBO``), but Chado can support a variety of different biological data types and there are often no community standard file formats for loading these data. For example, there is no file format for importing genotype and phenotype data. Those data can be stored in the feature, stock and natural diversity tables of Chado. The Bulk Loader was introduced in Tripal v1.1 and provides a web interface for building custom data loader. In short, the site developer creates the bulk loader "template". This template can then be used and re-used for any tab delimited file that follows the format described by the template. Additionally, bulk loading templates can be exported allowing Tripal sites to share loaders with one another.  Loading templates that have been shared are available on the Tripal website here: http://tripal.info/extensions/bulk-loader-templates.
+
+The following commands can be executed to install the Tripal Bulk Loader using Drush:
+
+.. code-block bash
+
+  cd /var/www/
+  drush pm-enable tripal_bulk_loader
+
+Plan How to Store Data
+----------------------
+
+To demonstrate use of the Bulk Loader, a brief example that imports a list of organisms and associates them with their NCBI taxonomy IDs will be performed. The input tab-delimited file will contains the list of all *Fragaria* (strawberry) species in NCBI at the time of the writing of this document.  Click the file link below and download it to ``/var/www/html/sites/default/files``.
+
+* `Fragaria.txt <http://tripal.info/sites/default/files/book_pages/Fragaria_0.txt>`_
+
+.. code-block bash
+
+  cd /var/www/html/sites/default/files
+  wget http://tripal.info/sites/default/files/book_pages/Fragaria_0.txt
+
+
+This file has three columns: NCBI taxonomy ID, genus and species:
+
+.. .. csv-table:: Fragaria sample file
+
+  3747    "Fragaria"        "x ananassa"
+  57918   "Fragaria"        "vesca"
+  60188   "Fragaria"        "nubicola"
+  64939   "Fragaria"        "iinumae"
+  64940   "Fragaria"        "moschata"
+  64941   "Fragaria"        "nilgerrensis"
+  64942   "Fragaria"        "viridis"
+
+
+To use the bulk loader you must be familiar with the Chado database schema and have an idea for where data should be stored. It is best practice to consult the GMOD website or consult the Chado community (via the `gmod-schema mailing list <https://lists.sourceforge.net/lists/listinfo/gmod-schema>`_) when deciding how to store data. For this example, we want to add the species to Chado, and we want to associate the NCBI taxonomy ID with these organisms. The first step, therefore, is to decide where in Chado these data should go. In Chado, organisms are stored in the **organism** table. This table has the following fields:
+
+`chado.organism Table Schema`
+
+.. csv-table::
+  :header: "Name",	"Type",	"Description"
+
+  "organism_id",	"serial",	"PRIMARY KEY"
+  "abbreviation",	"character varying(255)",
+  "genus",	"character varying(255)",	"UNIQUE#1 NOT NULL"
+  "species",	"character varying(255)",	"UNIQUE#1 NOT NULL  A type of organism is always uniquely identified by genus and species. When mapping from the NCBI taxonomy names.dmp file, this column must be used where it is present, as the common_name column is not always unique (e.g. environmental samples). If a particular strain or subspecies is to be represented, this is appended onto the species name. Follows standard NCBI taxonomy pattern."
+ 	"common_name",	"character varying(255)"
+ 	"comment",	"text"
+
+
+We can therefore store the second and third columns of the tab-delimited input file in the **genus** and **species** columns of the organism table.
+
+In order to store a database external reference (such as for the NCBI Taxonomy ID) we need to use the following tables: **db**, **dbxref**, and **organism_dbxref**. The **db** table will house the entry for the NCBI Taxonomy; the **dbxref** table will house the entry for the taxonomy ID; and the **organism_dbxref** table will link the taxonomy ID stored in the **dbxref** table with the organism housed in the **organism** table. For reference, the fields of these tables are as follows:
+
+
+`chado.db Table Schema`
+
+.. csv-table::
+  :header: "Name",	"Type",	"Description"
+
+ 	"db_id",	"serial",	"PRIMARY KEY"
+ 	"name",	character varying(255),	"UNIQUE NOT NULL"
+ 	"description",	"character varying(255)", ""
+ 	"urlprefix",	"character varying(255)"
+ 	"url",	"character varying(255)"
+
+
+`chado.dbxref Table Schema`
+
+.. csv-table::
+  :header: "Name",	"Type",	"Description"
+
+ 	"dbxref_id",	"serial",	"PRIMARY KEY"
+  "db_id",	"integer",	"Foreign Key db.  UNIQUE#1 NOT NULL"
+ 	"accession",	"character varying(255)",	"UNIQUE#1 NOT NULL.  The local part of the identifier. Guaranteed by the db authority to be unique for that db."
+ 	"version",	"character varying(255)",	"UNIQUE#1 NOT NULL DEFAULT ''"
+ 	"description",	"text"
+
+
+`chado.organism_dbxref Table Schema`
+
+.. csv-table::
+  :header: "Name",	"Type",	"Description"
+
+  "organism_dbxref_id", "serial", "PRIMARY KEY"
+  "organism_id",	"integer",	"Foreign key organism. UNIQUE#1 NOT NULL"
+  "dbxref_id",	"integer",	"Foreign key dbxref.  UNIQUE#1 NOT NULL"
+
+
+For our bulk loader template, we will therefore need to insert values into the **organism**, **db**, **dbxref** and **organism_dbxref** tables. In our input file we have the genus and species and taxonomy ID so we can import these with a bulk loader template. However, we do not have information that will go into the db table (e.g. "NCBI Taxonomy"). This is not a problem as the bulk loader can use existing data to help with import. We simply need to use the "NCBI Taxonomy" database that is currently in the Chado instance of Tripal v3.
+
+Creating a New Bulk Loader Template
+-----------------------------------
+
+Now that we know where all of the data in the input file will go and we have the necessary dependencies in the database (i.e. the NCBI Taxonomy database), we can create a new bulk loader template. Navigate to ``Tripal → Data Loaders → Chado Bulk Loader``, click the **Templates** tab in the top right corner, and finally click the link **Add Template**. The following page appears:
+
+.. image:: ./bulk_loader.1.png
+
+We need to first provide a name for our template. Try to name templates in a way that are meaningful for others. Currently only site administrators can load files using the bulk loader. But, future versions of Tripal will provide functionality to allow other privileged users the ability to use the bulk loader templates. Thus, it is important to name the templates so that others can easily identify the purpose of the template. For this example, enter the name **NCBI Taxonomy Importer (taxid, genus, species)**. The following page appears:
+
+.. image:: ./bulk_loader.2.png
+
+Notice that the page is divided into two sections: **Current Records** and **Current Fields**. Before we continue with the template we need a bit of explanation as to the terminology used by the bulk loader. A **record** refers to a Chado table and an action on that table. For example, to insert the data from the input file we will need to select the NCBI Taxonomy database from the **db** table and insert entries into the **dbxref**, **organism** and **dbxref_organism** tables. Therefore, we will have four records:
+
+* An insert into the organism table
+* A select from the db table (to get the database id (db_id) of the "NCBI Taxonomy" database needed for the insert into the dbxref table)
+* An insert into the dbxref table
+* An insert into the organism_dbxref table.
+
+Each record contains a set of fields on which the action is performed. Thus, when we insert an entry into the organism table we will insert into two fields: **genus** and **species**.
+
+To create the first record for inserting an organism, click the button **New Record/Field**. The following page appears:
+
+.. image:: ./bulk_loader.3.png
+
+By default, when adding a new record, the bulk loader also provides the form elements for adding the first field of the record as well. We are adding a new record, so we can leave the **Record** drop-down as **New Record**. Next, give this record a unique record name. Because we are inserting into the organism table, enter the name **Organism** into the **Unique Record Name** box.
+
+We also have the opportunity with this form to add our first field to the record.  Because we are adding the organism record we will first add the field for the **genus**. In the **Field** section we specify the source of the field. Because the genus value comes from the input file, select the first radio button titled **Data**. Next we need a human-readable name for the field. This field is the **genus** field so we will enter Genus into the **Human-readable Title for Field** box.  Next, we need to specify the **Chado table** for this record. In the Chado table drop down box, choose the **organism** table, and in the **Chado Field/Column** drop down box select **genus**.
+
+In the next section, titled **Data File Column**, we need to indicate the column in the tab-delimited file where the genus is found. For the example file this is column 2 (columns are ordered beginning with number 1). Therefore, enter the number **2** in the **Column** box. There are additional options to expose the field to the user, but for now we can ignore those options. Click the **Save Changes** button at the bottom. We now see that the organism record and the first field have been added to our bulk loader template.
+
+.. image:: ./bulk_loader.4.png
+
+We also see that the **Mode** (or action) for this record has been set to insert by default. Before continuing we should edit the settings for the record so that it is more fault tolerant. Click the **Edit** link to the left of the new organism record. On the resulting page we see the record details we already provided, but now there is a section titled **Action to take when Loading Record**. By default, the **INSERT** option is selected. This is correct. We want to perform an insert. However, notice in the **Additional Insert Options** section, the **SELECT if duplicate (no insert).** Check this box. This is a good option to add because it prevents the bulk loader from failing if the record already exists in the table.
+
+Click the **Save Record** button to save these settings. Now, you will see that the **Mode** is now set to insert or select if duplicate. Previously the **Mode** was just **insert**.
+
+Next, we need to add the **species** field to the record. Click the **Add Field** link to the left of the organism record name. Here we are presented with the same form we used when first adding the organism record. However, this time, the **Record** section is collapsed.  If we open that section the drop down already has the **Organism** record as we are not creating a new record. To add the **Species** field, provide the following values and click the **Save Changes button**:
+
+* Type of field: Data
+* Human-readable Title for Field: Species
+* Chado table: organism (should already be set)
+* Chado Field/Column: species
+* Column: 3
+
+We now have two fields for our organism record:
+
+.. image:: ./bulk_loader.5.png
+
+At this point our organism record is complete, however there are still a few fields in the organism table of Chado that are not present in our record. These include the **organism_id, abbreviation, common_name** and **comment** fields. We do not have values in our input file for any of these fields. Fortunately, the **organism_id** field is a primary key field and is auto generated when a record is submitted. We do not need to provide a value for that field. The other fields are not part of the unique constraint of the table. Therefore, those fields are optional and we do not need to specify them. Ideally, if we did have values for those non-required fields we would add them as well.
+
+To this point, we have built the loader such that it can load two of the three columns in our input file. We have one remaining column: the NCBI taxonomy ID. In order to associate an organism with the taxonomy ID we must first insert the taxonomy ID into the **dbxref** table. Examining the dbxref table, we see that a **db_id** field is a required value in a foreign key relationship. We must first retrieve the **db_id** from the **db** table of Chado before we can add the entry to the **dbxref** table. Therefore, we will create a second record that will do just that. On the **Edit Template** page click the button **New Record/Field**. Here we see the same form we used for adding the first organism record. Provide the following values:
+
+* For the record:
+   * Record: New Record
+   * Unique Record Name: NCBI Taxonomy DB
+   * Record Type/Action: SELECT ONCE: Select the record only once for each constant set.
+* For the field:
+   * Type of field: Constant
+   * Human-readable Title for Field: DB name
+   * Chado table: db
+   * Chado field/column: name
+* Within the Constant section:
+   * Constant Value:  NCBITaxon
+   * Check "Ensure the value is in the table"
+
+Here we use a field type of **Constant** rather than **Data**. This is because we are providing the value to be used in the record rather than using a value from the input file. The value we are providing is "NCBI Taxonomy" which is the name of the database we added previously. The goal is to match the name "NCBI Taxonomy" with an entry in the **db** table. Click the **Save Changes** button.
+
+We now see a second record on the **Edit Template** page. However, the mode for this record is insert. We do not want to insert this value into the table, we want to select it because we need the corresponding **db_id** for the **dbxref** record. To change this, click the Edit link to the left of the **NCBI Taxonomy DB** record. Here we want to select only the option **SELECT ONCE**. We choose this option because the database entry that will be returned by the record will apply for the entire input file. Therefore, we only need to select it one time. Otherwise, the select statement would execute for each row in the input file causing excess queries. Finally, click **Save Record**. The **NCBI Taxonomy DB** record now has a mode of **select once**.  When we created the record, we selected the option to 'SELECT ONCE'.  This means that the bulk loader will perform the action one time for that record for the entire import process.  Because the field is a constant the bulk loader need not execute that record for every row it imports from our input file.  We simply need to select the record once and the record then becomes available for use through the entire import process.
+
+Now that we have a record that selects the **db_id** we can now create the **dbxref** record. For the **dbxref** record there is a unique constraint that requires the **accession**, **db_id** and **version**. The version record has a default value so we only need to create two fields for this new record: the db_id and the accession. We will use the **db_id** from the **NCBI Taxonomy DB** record and the accession is the first column of the input file. First, we will add the **db_id** record. Click the **New Record/Field** button and set the following:
+
+* For the record:
+   * Record: New Record
+   * Unique Record Name: Taxonomy ID
+   * Record Type/Action:  INSERT: insert the record
+* For the field:
+   * Type of field: Record referral
+   * Human-readable Title for Field: NCBI Taxonomy DB ID
+   * Chado table: dbxref
+   * Chado Field/Column: db_id
+* In the Record Referral Section:
+   * Record to refer to: NCBI Taxonomy DB
+   * Field to refer to: db_id
+
+
+Click the Save Changes button. The Edit Template page appears.
+
+.. image:: ./bulk_loader.6.png
+
+Again, we need to edit the record to make the loader more fault tolerant. Click the Edit link to the left of the Taxonomy ID record. Select the following:
+
+* Insert
+* Select if duplicate
+
+To complete this record, we need to add the accession field. Click the Add field link to the left of the Taxonomy ID record name. Provide the following values:
+
+* For the field:
+   * Type of Field: Data
+   * Human-readable Title for Field: Accession
+   * Chado table: dbxref
+   * Chado field/column: accession
+* In the Data File Column section:
+   * Column: 1
+
+At this state, we should have three records: Organism, NCBI Taxonomy DB, and Taxonomy ID. We can now add the final record that will insert a record into the **organism_dbxref** table. Create this new record with the following details:
+
+* For the record:
+   * Record: New Record
+   * Unique Record Name: Taxonomy/Organism Linker
+   * Check: Insert: insert the record
+* For the field:
+   * Type of Field: Record Referral
+   * Human-readable Title for Field: Accession Ref
+   * Chado table: organism_dbxref
+   * Chado field/column: dbxref_id
+* In the Record Referral section:
+   * Record to refer to: Taxonomy ID
+   * Field to refer to: dbxref_id
+
+Create the second field:
+
+* For the field:
+   * Type of Field: Record Referral
+   * Human-readable Title for Field: Organism ID
+   * Chado table: organism_dbxref
+   * Chado field/column: organism_id
+* In the Record Referral section:
+   * Record to refer to: Organism
+   * Field to refer to: organism_id
+
+​After saving the field.  Edit the record and set the following:
+
+* Change the record mode to: insert or select if duplicate
+
+We are now done! We have created a bulk loader template that reads in a file with three columns containing an NCBI taxonomy ID, a genus and species. The loader places the genus and species in the **organism** table, adds the NCBI Taxonomy ID to the **dbxref** table,  links it to the NCBI Taxonomy entry in the db table, and then adds an entry to the **organism_dbxref** table that links the organism to the NCBI taxonomy Id. The following screen shots show how the template should appear:
+
+.. image:: ./bulk_loader.7.png
+
+To save the template, click the **Save Template** link at the bottom of the page.
+
+Creating a Bulk Loader Job (importing a file)
+---------------------------------------------
+
+Now that we have created a bulk loader template we can use it to import a file. We will import the **Fragaria**.txt file downloaded previously. To import a file using a bulk loader template, click the **Add Content** link in the administrative menu and click the **Bulk Loading Job**. A bulk loading job is required each time we want to load a file. Below is a screen shot of the page used for creating a bulk loading job.
+
+.. image:: ./bulk_loader.8.png
+
+Provide the following values:
+
+* Job Name: Import of Fragaria species
+* Template: NCBI Taxonomy Importer (taxid, genus species).
+* Data File: /var/www/html/sites/default/files/Fragaria_0.txt
+* Keep track of inserted IDs: No
+* File has a header: No
+
+Click **Save**. The page then appears as follows:
+
+.. image:: ./bulk_loader.9.png
+
+You can see details about constants that are used by the template and the where the fields from the input file will be stored by clicking the **Data Fields** tab in the table of contents on the left sidebar.
+
+.. image:: ./bulk_loader.10.png
+
+Now that we have created a job, we can submit it for execution by clicking the **Submit Job** button. This adds a job to the Tripal Jobs systems and we can launc the job as we have previously in this tutorial:
+
+.. code-block:: shell
+
+  cd /var/www
+  drush trp-run-jobs --username=admin --root=/var/www/html
+
+After execution of the job you should see similar output to the terminal window:
+
+.. code-block:: shell
+
+
+  Tripal Job Launcher
+  Running as user 'admin'
+  -------------------
+  There are 1 jobs queued.
+  Calling: tripal_bulk_loader_load_data(2, 7)
+  Template: NCBI Taxonomy Importer (taxid, genus, species) (1)
+  File: /var/www/html/sites/default/files/Fragaria_0.txt (46 lines)
+
+  Preparing to load...
+  Loading...
+      Preparing to load the current constant set...
+          Open File...
+          Start Transaction...
+          Defer Constraints...
+          Acquiring Table Locks...
+              ROW EXCLUSIVE for organism
+              ROW EXCLUSIVE for dbxref
+              ROW EXCLUSIVE for organism_dbxref
+      Loading the current constant set...
+  Progress:
+  [|||||||||||||||||||||||||||||||||||||||||||||||||||] 100.00%. (46 of 46) Memory: 33962080
+
+Our *Fragaira* species should now be loaded, and we return to the Tripal site to see them. Click on the **Organisms** link in the **Search Data** menu.  In the search form that appears, type "Fragaria" in the **Genus** text box and click the **Filter** button. We should see the list of newly added *Fragaria* species.
+
+.. image:: ./bulk_loader.11.png
+
+Before the organisms will have Tripal pages, the Chado records need to be **Published**.  You can publish them by navigating to **Tripal Content -> Publish Tripal Content**.  Select the **organism** table from the dropdown and run the job.
+
+.. note::
+
+	In Tripal 2, records were synced by naviating to **Tripal → Chado Modules → Organisms**.
+
+Once complete, return to the search form, find a *Fragaria* species that has been published and view its page. You should see a Cross References link in the left table of contents. If you click that link you should see the NCBI Taxonomy ID with a link to the page:
+
+.. image:: ./bulk_loader.12.png
+
+
+Sharing Your Templates with Others
+----------------------------------
+
+Now that our template for loading organisms with NCBI Taxonomy IDs is completed we can share our template loader with anyone else that has a Tripal-based site.  To do this we simply export the template in text format, place it in a text file or directly in an email and send to a collaborator for import into their site.  To do this, navigate to **Tripal → Chado Data Loaders → Buik Loader** and click the **Tempalate** tab at the top.  Here we find a table of all the tempaltes we have created.  We should see our template named **NCBI Taxonomy Importer** (taxid, genus, species).  In the far right colum is a link to export that template.  Licking that link will redirect you to a page where the template is provided in a serialized PHP array.
+
+.. image:: ./bulk_loader.13.png
+
+Cut-and-paste all of the text in the **Export** field and send it to a collaborator.
+
+To import a template that may have been created by someone else, navigate to **Tripal → Chado Data Loaders → Buik Loader** and click the **Tempalate** tab. A link titled Import Template appears above the table of existing importers.  The page that appears when that link is clicked will allow you to import any template shared with you.

docs/user_guide/configuring_page_display.rst

@@ -0,0 +1,57 @@
+
+Configuring Page Display
+=========================
+
+
+This is one of the many new exciting features of Tripal v3.x. In this version of Tripal we have taken integration with Drupal Fields to a whole new level representing each piece of content (in Chado or otherwise) as a Drupal Field. What this means for site builders is unprecendented control over content display and arrangement through the administrative user interface --No more editing PHP template files to change the order, grouping or wording of content!
+
+You can configure the display of a given Tripal Content Type by navigating to **Structure → Tripal Content Types** and then selecting the **Manage Display** link beside the content type you would like to configure.
+
+.. image:: ./configuring_page_display.1.png
+
+
+
+The Manage Display User Interface lists each Drupal Field in the order they will be displayed on the page. Fields are grouped into Tripal Panes by the Tripal DS module and the page is automatically divided into a right and left column. By default the left column contains the table of contents which lists the Tripal Panes available to the user in the order they are listed in this UI. The following screenshots are using the Analysis Content Type for demonstatration.
+
+.. image:: configuring_page_display.2.png
+
+
+Rearranging Fields
+------------------
+
+To rearrange the fields within a Tripal pane, simply drag them into the order you would like them. For example, the description is currently within the Summary table --it makes much more sense for it to be below the table but still within the summary. To do this, simply drag the description field to the bottom of the summary table and then move it in one level as shown in the following screenshot. Then click the **Save** button at the botton to save the changes.
+
+.. image:: configuring_page_display.3.rearrange.png
+
+
+Removing Fields and/or Field Labels
+-----------------------------------
+
+Now say we don't want the label "Description" in front of description content since it's pretty self explanatory. We can do that by changing the drop-down beside "Description" which currently says "Above" to "Hidden". This removes the label for the field assuming it's not within a table.
+
+There may also be data you want to collect from your user but don't want to display on the page. This can be achomplished by disabling the field in the Manage Display UI. For example, we might not feel the need to tell users that this is an alaysis page and thus want to hide the Resource Type Field. This is done by changing the drop-down beside the Resource type field from "Right" to "Disabled".
+
+.. warning::
+
+  Don't forget to save the configuration often as you are changing it. You will not see changes to the page unless the **Save** button at the bottom of the Manage Display UI is clicked.
+
+
+Changing Tripal Pane Names
+--------------------------
+
+The name of a Tripal Pane is displayed both in the header of the Pane itself and in the Table of Contents. To change this name, click the gear button to the far right of the Tripal Pane you would like to change. This will bring up a blue pane of settings. Changing the Field Group Label will change the display name of the pane. For example, the following screenshot shows how you would change the "Cross References" Tripal Pane to be labeled "External Resources" instead if that it what you prefer. Then just click the Update button to see your changes take effect.
+
+.. image:: ./configuring_page_display.4.png
+
+
+Display/Hide Tripal Panes on Page Load
+--------------------------------------
+
+You can also easily control which Tripal Panes you would like displayed to the user on initial page load. By default the Summary Pane is the only one configured to show by default. However, if you would prefer for all panes or even a specific subset of panes to show by default, you can simply click the gear button to the far right of each Tripal Pane you want displayed by default and uncheck the "Hide panel on page load" checkbox. This gives you complete control over which panes you want your user to see first. If more then one pane is displayed by default then they will be shown in the order they are listed on the Manage Display UI.
+
+Display/Hide Empty Fields
+-------------------------
+
+By default Tripal v3 hides all empty fields from the user. However like most behaviour in Tripal, this can be configured. If you would prefer to show all fields to the user regardless of whether there is content for that particular page, then navigate to ``Structure → Tripal Content Types`` and then click on the edit link beside the Tripal Content Type you would like to show empty fields for. Near the bottom of this form is a **Field Display** drop-down. Just change this drop-down to "show empty fields" and then click **Save Content Type**. As an example, we have changed this setting for the organism content type and, as you can see below, now you can see all fields (including empty fields like cross references and relationships) available to the organism content type.
+
+.. image:: ./configuring_page_display.5.png

docs/user_guide/creating_content.rst

@@ -0,0 +1,63 @@
+Creating Content Types
+======================
+
+.. note::
+
+  Prior to creating a new content type you should understand the structure of Chado and how others use Chado to store similar types of data.
+
+
+Tripal v3 comes with some pre-defined content types, however you have the ability to create new Content Types through the Administrative user interface! In Tripal v3, all content types are defined by `Controlled Vocabulary (CV) terms <https://en.wikipedia.org/wiki/Controlled_vocabulary>`_. This has a number of advantages:
+
+1. Facilitates sharing between Tripal sites.
+2. Provides a clear indication of what content is available on your site.
+3. Makes content creation more intuitive from Tripal v2 (add a "Gene" rather then a "feature").
+4. Allows complete customization of what data types your site provides.
+5. Integrates tightly with web services allowing Tripal to adhere to RDF specifications.
+
+Find a Controlled Vocabulary (CV) Term
+---------------------------------------
+
+Before creating a new content type for your site you must identify a CV term that best matches the content type you would like to create.  CVs are plentiful and at times selection of the correct term from the right vocabulary can be challenging. If there is any doubt about what term to use, then it is best practice to reach out to others to confirm your selection. The Tripal User community is a great place to do this by posting a description of your content type and your proposed term on the `Tripal Issue Queue <https://github.com/tripal/tripal/issues>`_.  Confirming your term with others will also encourage re-use across Tripal sites and improve data exchagnge capabilities.
+
+The `EBI's Ontology Lookup Service <http://www.ebi.ac.uk/ols/index>`_ is a great place to locate terms from public vocabularies. At this site you can search for terms for your content type.  If you can not find an appropriate term in a public vocabulary or via discussion with others then you create a new **local** term within the **local** vocabulary that comes with Tripal.
+
+.. warning::
+
+  Creation of **local** terms is discouraged but sometimes necessary.  When creating local terms, be careful in your description.
+
+How to Add a CV Term
+--------------------
+Loading From an OBO File
+^^^^^^^^^^^^^^^^^^^^^^^^
+Once you've choosen a term to describe your content type, you may need to add the term to Tripal if it is not already present.  Many CVs use the `OBO file format <https://owlcollab.github.io/oboformat/doc/GO.format.obo-1_4.html>`_ to define their terms. If the term belongs to a controlled vocabulary with a file in OBO format then you can load all the terms of the vocabulary using Tripal's OBO Loader at **Tripal → Data Loaders → Chado Vocabularies → Chado OBO Loader**.
+
+Manually Adding a Term
+^^^^^^^^^^^^^^^^^^^^^^
+Alternatively, you can add terms one at a time. To add a single term either from an existing vocabulary or a new local term, navigate to **Tripal → Data Loaders → Chado Vocabularies → Manage Chado CVs** and search to see if the vocabulary already exists. If it does you do not need to add the vocabulary.  If it does not exist, click the **Add Vocabulary** link to add the vocabulary for your term. Then navigate to **Tripal → Data Loaders → Chado Vocabularies → Mange Chado CV Terms** then click the **Add Term link** to add the term.
+
+Create a Tripal Content Type
+----------------------------
+
+Creation of a new content type requires familiarity with Chado.  This is because data records used by content types must be mapped to actual data and the data lives in Chado.  Tripal's interface for creating content types allows you to provide the CV term for the type and then indicate where in Chado the data is/will be stored.  Chado is a flexible relational database schema.  Thus, it is possible for different sites to store data in different ways.  It is best practice however to follow community standards when storing data.  Therefore, please review the online documentation for Chado. If you are unclear how data for your content type should be stored in Chado please consider emailing the `Chado mailing list <http://gmod.org/wiki/GMOD_Mailing_Lists>`_ to ask for help or add a request for help on the Tripal issue queue.
+
+To add a new content type, start by navigating to **Structure → Tripal Content Types** and  click on the **Add Tripal Content Type** link at the top. This will take you to a web form that leads you through the process of creating a custom Tripal Content Type. First, enter for the name of the term you would like to use to describe your content in the Content Type autocomplete textbox (e.g. genetic_marker). Then, click **Lookup Term**. This should bring up a list of matching terms from which you can select the specific term you would like to use.  Sometimes the same term exists in multiple vocabularies and you can select the proper one.
+
+.. image:: creating_content.create1.png
+
+During content type creation there is as a section to specify which Chado tables will store your data. Chado is typically structured with primary **base** tables (e.g. organism, feature, stock, project, etc) and a set of linker and property tables that contain ancillary data related to the base records.  Here you must first choose the base table where primary records for your data type are stored.  For our example, because genetic markers are sequence features, they are stored in the Chado feature table. Once you select the Chado table, the form will ask additional questions to determine exactly how records of the content type can be found. Specifically, the following options are supported if the appropriate fields/tables are available:
+
+1. All records in the **base** table belong to the content type (e.g. tables: organism, analysis, etc.)
+2. The **base** table has a **type_id** that stores the CV term and this differentiates the records. (e.g. tables: feature, stock, library, etc.).
+3. The records can be differentiated by way of a property table which contains a **type_id** column for the CV term. (e.g. tables: featureprop, stockprop, libraryprop, etc.)
+4. The records can be differentiated by way of a linking table that associates records in the **base** table with the CV term (e.g. tables: feature_cvterm, stock_cvterm, etc.)
+
+For our genetic marker example, we can use the Chado **feature** table and **type_id** column to differentiate which records in the feature table are genetic markers. Thus we
+
+- Select "No" not all records in the feature table are genetic markers
+- Type Column: type_id
+
+Then click Create Content Type to create a custom genetic marker content type.
+
+.. image:: creating_content.create2.png
+
+Once the content type is created, you can create pages for site visitors. This will be described later in this User's Guide. In short, you can manually create new records through brand new web forms that are created automatically for your content type, or you can use a data loader to import your data directly to Chado, then **Publish** those records through the Tripal interface.

docs/user_guide/customize_site.rst

@@ -0,0 +1,4 @@
+Customizing Your Site
+=====================
+
+Through the combination of Drupal, Chado and the Tripal API, it is possible to fully customize your site and to add new functionality.  Tripal does not store or display all data out-of-the box, and every site has its own different look-and-feel.  If you want greater functionality beyond what comes with Tripal please see the :doc:`Developer's Handbook </dev_guide>` for instructions to use the Drupal and Tripal APIs to develop your own extensions to Tripal.

docs/user_guide/drupal_overview.rst

@@ -0,0 +1,146 @@
+Brief Drupal Overview
+=====================
+
+.. note::
+
+  Some of the images used on this page refer to Tripal v2.  However, this Drupal overview applies equally well to both versions.
+
+User Account Page
+-----------------
+
+All users have an account page. When you first install Drupal, you are logged in as the administrator. The account page is simple for now. Click the **My account** link on the left sidebar. You'll see a brief history for the user and an **Edit** tab. Users can edit their own information using the edit interface:
+
+.. image:: drupal_overview.account_edit.png
+
+Creating Content
+----------------
+
+Creation of content in Drupal is very easy. Click the **Add content** link on the top administrative menu.
+
+.. image:: drupal_overview.create_content1.png
+
+You'll see two content types that come default with Drupal: Article and Basic Page. Here is where a user can add simple new pages to the website without knowledge of HTML or CSS. Click the **Basic Page** content type to see the interface for creating a new page:
+
+.. image:: drupal_overview.create_content2.png
+
+You'll notice at the top a **Title** field and a **Body** text box. All pages require a title and typically have some sort of content entered in the body. Additionally, there are other options that allow someone to enter HTML if they would like, save revisions of a page to preserve a history and to set authoring and publishing information.
+
+For practice, try to create two new pages. A **Home** page and an **About** page for our site. First, create the home page and second create the about page. Add whatever text you like for the body.
+
+Finding Content
+---------------
+
+To find any content that has been created on the site, click the **Find Content** link on the administrative menu at the top. The page shows all content available on the site. You will see the **About** and **Home** pages you created previously:
+
+.. image:: drupal_overview.find_content.png
+
+You'll also notice a set of drop down boxes for filtering the content. For sites with many different content types and pages this helps to find content. You can use this list to click to view each page or to edit.
+
+Site Administration
+-------------------
+
+Modules
+^^^^^^^
+Click the **Modules** link on administrative menu at the top of the page:
+
+.. image:: drupal_overview.modules.png
+
+Here is where you see the various modules that make up Drupal. Take a minute to scroll through the list and read some of the descriptions. The modules you see here are core modules that come with Drupal. Those that are checked come pre-enabled. Those that are not checked we will need to install them if we want to use them. To enable or "turn on" a module, check the box next to the desired module, then scroll to the bottom and click 'Save configuration'. Your site will now have the functionality provided by that module. Alternatively, you can search for modules that may be useful to your intended site design at the Drupal module repository, https://drupal.org/project/project_module, and install them by clicking the **Install New Module** link. Finally, a 3rd method to install modules is by use of the drush tool. We will use drush for this tutorial.
+
+Themes
+^^^^^^
+Next, click the **Appearance** link on the administrative menu at the top of the page:
+
+.. image:: drupal_overview.appearance1.png
+
+Here, you'll see a list of themes that come with Drupal by default. Here you will see the **default theme** is called **Bartik**. This theme controls the appearance of all content on the site. You can easily change the way the site looks by changing the default theme to another theme. For this tutorial, we would like to use the **Garland** theme. If you scroll down you'll see that one theme named Garland. click the link in the Garland theme section titled **Enable and set default**. The current look of the site is using the Garland theme.
+
+.. image:: drupal_overview.appearance2.png
+
+Now, click the house icon in the top left. Our home page now uses the Garland theme:
+
+.. image:: drupal_overview.appearance3.png
+
+Blocks
+^^^^^^
+Blocks in Drupal are used to provide additional content to any page that already exists. Examples of blocks might be a short overview of recent news items, Twitter feeds, links, recently added content, etc. The blocks interface can be found by navigating to **Structure → Blocks** using the top administrative menu.
+
+On this page you'll see a list of available blocks and where they are located within the site.
+
+.. image:: drupal_overview.blocks1.png
+
+Here you can see that the **Search form**, **Navigation**, and **User Login** blocks are all on the left sidebar of Garland theme. There are also a list of other regions available that do not have any blocks and there are many blocks which are Disabled but could be added to a region on the page. For this tutorial, we would like for blocks to appear on the right sidebar rather than the left sidebar. Therefore, change the **Search form**, **Navigation**, and **User Login** to all use the right sidebar by changing the drop down box next to each one. When done, click the **Save Blocks** button at the bottom. Now when we view our home page the navigation links, search form and user login box (not shown while logged in) all appear on the right side:
+
+.. image:: drupal_overview.blocks2.png
+
+Menus
+^^^^^
+For this tutorial, we want to add new links in the **Main Menu** to our new Home and About pages we created earlier. In the Garland theme, the main menu appears in the top right corner and currently only has the link 'Home'. We want to change this link to direct to our new home page. But first, we need to find the path for our home page. The path for a page can be found in the address bar for the page. In Drupal pages of content are generally referred to as **nodes**. We can find the new home and about pages using the **Find content** link in the top administrative menu. If we click the link for our home page you'll see the address is http://localhost/node/1. Our about page is http://localhost/node/2 (i.e the first and second pages we created).
+
+Drupal provides an interface for working with menus, including adding new menu items to an existing menu or for creating new menus. You can find the interface for working with menus by navigating to **Structure → Menus** via the administrative top menu:
+
+.. image:: drupal_overview.menus1.png
+
+Click the link list links in the operations section for the **Main Menu**. Here we see that the **Home** link already exists:
+
+.. image:: drupal_overview.menus2.png
+
+Click edit to change the location of the Home menu item. In the form that appears, we need to set the path for our new home page. The path for each of these nodes is **node/1** and **node/2**. Fill out the form fields with these values
+
+.. csv-table::
+  :header: "Form element", "Value"
+
+  "Menu Link Title", "Home"
+  "Path", "node/1"
+  "Description", "Tripal 2.0 Demo Home Page"
+  "Enabled", "checked"
+  "Show as Expanded", "no check"
+  "Parent item", "<Main menu>"
+  "Weight", "0"
+
+The resulting page appears as follows:
+
+.. image:: drupal_overview.menus3.png
+
+The settings above will give the menu link a title of **Home** and put it on the Main menu menu. If we then click the **Save** button at the bottom our **Home** menu item now redirects us to our new home page. Now, we also want to add a new menu item for the **About** page. Return to the **Main menu** configuration page and add a new link with the following values:
+
+.. csv-table::
+  :header: "Form element", "Value"
+
+  "Menu Link Title", "About"
+  "Path", "node/2"
+  "Description", "About this site"
+  "Enabled", "checked"
+  "Show as Expanded", "no check"
+  "Parent item", "<Main menu>"
+  "Weight", "0"
+
+Click **Save** and a new menu item should appear. You can then change the order of the menu items by dragging and dropping the link using the cross-hairs next to each menu item.
+
+URL Path
+^^^^^^^^
+As mentioned previously, the URL paths for our pages have node/1 and node/2 in the address. This is not very intuitive for site visitors.
+
+To set a path, click on our new **About** page in the new menu link at the top and click the **Edit** tab (you may have to close the overlay to see the menu item). Scroll to the bottom of the edit page and you'll see a section titled **URL path setting**. click to open this section. Since this is our about page, we simply want the URL to be http://localhost/about. To do this, just add the word "about" in the text box and click the **Save** button. You will now notice that the URL for this page is no longer http://localhost/node/2 but now http://localhost/about. Although, both links will still get you to our About page.
+
+.. image:: drupal_overview.create_page.png
+
+Now, use the instructions described above to set a path of 'home' for our home page.
+
+Site Configuration
+^^^^^^^^^^^^^^^^^^
+There are many options under the **Configuration** link of the administrative menu at the top. Here we will only look at one of these at the moment--the **Site Information** page. Here you will find the settings we made when installing the site. You can change the site name, add a slogan, mission and footer text to the. The section titled **Front Page** is where we can tell Drupal to use our new **Home** page we created as the first page visitors see when they view the site. We want this to be the same as the home page we created and added a link for in the **Main menu**. In this text box enter the text **node/1**. Notice there is no preceding forward slash. Alternatively we could have used the URL path we added in the previous step. Let's add a slogan: **Resources for Community Genomics**.
+
+.. image:: drupal_overview.settings.png
+
+Now, click the **Save configuration** button at the bottom. You'll now see the slogan now at the top of the page. Also, if you click the site name or the home icon at the top left we are now redirected to the new home page.
+
+User Accounts
+-------------
+For this tutorial, we will not discuss in depth the user management infrastructure except to point out:
+
+- User accounts can be created
+- Users are assigned to various roles
+- Permissions for those roles can be set to allow groups of users certain administrative rights or access to specific data.
+
+Explore the Drupal **User Management** menu to see how users can be created, added to roles with specific permissions.

docs/user_guide/example_genomics.rst

@@ -0,0 +1,16 @@
+Setup of an Example Site
+=================================
+
+The following tutorial will walk you through creating content and loading genomic data. This is a good introduction to Tripal v3.x Content Types and the new Administrative User Interface regardless of whether you intend to store genomic data in your particular Tripal v3 site.
+
+.. toctree::
+   :maxdepth: 1
+   :caption: Table of Contents
+   :glob:
+
+   ./example_genomics/organisms
+   ./example_genomics/analyses
+   ./example_genomics/cross_refs
+   ./example_genomics/controlled_vocabs
+   ./example_genomics/genomes_genes
+   ./example_genomics/pub_import

docs/user_guide/example_genomics/analyses.rst

@@ -0,0 +1,35 @@
+Analyses
+========
+
+
+For this tutorial we will later import a set of genes, and their associated mRNA, CDS, UTRs, etc. Tripal's Chado loader for importing genomic data requires that an analysis be associated with all imported features. This has several advantages, including:
+
+- The source of features (sequences) can be traced. Even for features simply downloaded from a database, someone else can see where the features came from.
+- Provenance describing how the features were created can be provided (e.g. whole genome structural and functional annotation description).
+- The analysis associates all of the features together.
+
+To create an analysis for loading our genomic data, navigate to the Add Tripal Content and click on the link: **Analysis**
+
+The analysis creation page will appear:
+
+.. image:: analyses.1.png
+
+Here you can provide the necessary details to help others understand the source of your data. For this tutorial, enter the following:
+
+.. csv-table::
+  :header: "Form Element",	"Value"
+
+  "Name", "Whole Genome Assembly and Annotation of Citrus Sinensis (JGI)"
+  "Program, Pipeline Name or Method Name", "Assembly and Annotation Performed by JGI"
+  "Program Version", "Phytozome v9"
+  "Time Executed", "For this tutorial just select any date."
+  "Data Source Name", "JGI Citrus sinensis assembly/annotation v1.0 (154)"
+  "Data Source URI", "http://www.phytozome.net/citrus.php"
+  "Description (Set to Full HTML):", "<p> <strong><em>Note: </em>The following text comes from phytozome.org:</strong></p> <p> <u>Genome Size / Loci</u><br /> This version (v.1) of the assembly is 319 Mb spread over 12,574 scaffolds. Half the genome is accounted for by 236 scaffolds 251 kb or longer. The current gene set (orange1.1) integrates 3.8 million ESTs with homology and ab initio-based gene predictions (see below). 25,376 protein-coding loci have been predicted, each with a primary transcript. An additional 20,771 alternative transcripts have been predicted, generating a total of 46,147 transcripts. 16,318 primary transcripts have EST support over at least 50% of their length. Two-fifths of the primary transcripts (10,813) have EST support over 100% of their length.</p> <p> <u>Sequencing Method</u><br /> Genomic sequence was generated using a whole genome shotgun approach with 2Gb sequence coming from GS FLX Titanium; 2.4 Gb from FLX Standard; 440 Mb from Sanger paired-end libraries; 2.0 Gb from 454 paired-end libraries</p> <p> <u>Assembly Method</u><br /> The 25.5 million 454 reads and 623k Sanger sequence reads were generated by a collaborative effort by 454 Life Sciences, University of Florida and JGI. The assembly was generated by Brian Desany at 454 Life Sciences using the Newbler assembler.</p> <p> <u>Identification of Repeats</u><br /> A de novo repeat library was made by running RepeatModeler (Arian Smit, Robert Hubley) on the genome to produce a library of repeat sequences. Sequences with Pfam domains associated with non-TE functions were removed from the library of repeat sequences and the library was then used to mask 31% of the genome with RepeatMasker.</p> <p> <u>EST Alignments</u><br /> We aligned the sweet orange EST sequences using Brian Haas's PASA pipeline which aligns ESTs to the best place in the genome via gmap, then filters hits to ensure proper splice boundaries.</p>"
+
+.. note::
+  Above, the description is provided as HTML code.  However if you enabled the **ckeditor** module (as instructed in the Tripal Prerequisites section), you should click the link **Switch to plain-text editor** found below the Description field before cut-and-pasting the code above.  Normally, you would enter the text free-hand but for this tutorial it is fastest to cut-and-paste the HTML.
+
+After saving, you should have the following analysis page:
+
+.. image:: analyses.2.png

docs/user_guide/example_genomics/controlled_vocabs.rst

@@ -0,0 +1,19 @@
+Controlled Vocabularies
+=======================
+
+Before we proceed with setup of our example genomics site we will want to load the Gene Ontology.  This is because we will be loading a whole genome, genes and transcripts with annotations.  These annotations include Gene Ontology terms.  To load the Gene Ontolgoy, navigate to **Tripal → Data Loaders → Chado Vocabularies → OBO Vocabulary Loader**. You will see the following page:
+
+.. image:: controlled_vocabs.1.png
+
+The Ontology loader allows you to select a pre-defined vocabulary for loading or allow you to provide your own. If you provide your own, you give the remote URL of the OBO file or provide the full path on the local web server where the OBO file is located. In the case of a remote URL, Tripal first downloads and then parses the OBO file for loading. If you do provide your own OBO file it will appear in the saved drop down list for loading of future updates to the ontology.
+
+During the Tripal installation portion of this tutorial, several vocabularies were pre-installed for you. The Gene Ontology, however, was not. To import the Gene Ontology, select it from the drop-down and click the Import Vocabulary button. You will notice a job is added to the jobs system. Now manually launch the jobs
+
+::
+
+  drush trp-run-jobs --username=administrator --root=/var/www/html
+
+
+.. note::
+
+  Loading the Gene Ontology will take several hours.

docs/user_guide/example_genomics/cross_refs.rst

@@ -0,0 +1,24 @@
+Cross References
+================
+
+For our gene pages and mRNA pages we want to link back to JGI where we obtained the genes. Therefore, we want to add a database reference for JGI. To add a new external databases, navigate to **Tripal → Data Loaders →  Chado Databases** and click the link titled **Add a Database**. The resulting page provides fields for adding a new database:
+
+.. image:: cross_refs.1.png
+
+Enter the following values for the fields:
+
+.. csv-table::
+  :header: "Field Name", "Value"
+
+  "Database Name", "Phytozome"
+  "Description", "Phytozome is a joint project of the Department of Energy's Joint Genome Institute and the Center for Integrative Genomics to facilitate comparative genomic studies amongst green plants"
+  "URL", "http://www.phytozome.net/"
+  "URL prefix", "https://phytozome.jgi.doe.gov/phytomine/portal.do?externalid=PAC:{accession}"
+
+The URL prefix is important as it will be used to create the links on our gene pages. When an object (e.g. gene) is present in another database, typically those database have a unique identifier (or accession) for the resource.  If we want to link records in our database to records in the remote database we need to provide a URL prefix that Tripal will use to create the URL.   Typically a remote database has a standard URL schema by which someone can specify a unique resource.  Often the resource accession is the last word in the URL to allow others to easily build the URL for any resource.  Tripal can take advantage of these type URL schemas via the URL Prefix field.
+
+The URL prefix should be the URL used to identify a resource.  Two tokens, {db} and {accession}, can be used in place of where the database name and accession might be needed to create the URL. If no {db} or {accession} are provided in the URL prefix then Tripal will append the database name and the accession to the URL prefix to form the final URL.  In this example, the Phytozome URL only requires the accession. The position where that accession will be placed is indicated with the {accession} token.  The {db} token is not needed.
+
+Click **Add**.
+
+We now have added a new database!

docs/user_guide/example_genomics/genomes_genes.rst

@@ -0,0 +1,223 @@
+Genomes and Genes
+=================
+
+Loading Feature Data
+--------------------
+Now that we have our organism and whole genome analysis ready, we can begin loading genomic data. For this tutorial only a single gene from sweet orange will be loaded into the databsae. This is to ensure we can move through the tutorial rather quickly. The following datasets will be used for this tutorial:
+
+- `Citrus sinensis-orange1.1g015632m.g.gff3 <http://tripal.info/sites/default/files/Citrus_sinensis-orange1.1g015632m.g.gff3>`_
+- `Citrus sinensis-scaffold00001.fasta <http://tripal.info/sites/default/files/Citrus_sinensis-scaffold00001.fasta>`_
+- `Citrus sinensis-orange1.1g015632m.g.fasta <http://tripal.info/sites/default/files/Citrus_sinensis-orange1.1g015632m.g.fasta>`_
+
+One of the new features available in many of the Tripal v3 data loaders is an HTML5 file upload element which allows administrators and users to upload large files reliably. This removes the requirement in previous versions of this tutorial to download these files directly on the server and provide a path to the file. Instead, if you have the file on your current local machine you can now simply upload it for loading.
+
+Another new option in Tripal v3 Data Loaders is the ability to provide a remote path of a file to be loaded. This completely alleviates the need to transfer large files multiple times and eases the loading process.
+
+Loading a GFF3 File
+-------------------
+The gene features (e.g. gene, mRNA, 5_prime_UTRs, CDS 3_prime_UTRS) are stored in the GFF3 file downloaded in the previous step. We will load this GFF3 file and consequently load our gene features into the database. Navigate to **Tripal → Data Loaders → Chado GFF3 Loader**.
+
+.. image:: genomes_genes.1.png
+
+Enter the following:
+
+.. csv-table::
+  :header: "Field Name", "Value"
+
+  "File", "Upload the file name Citrus_sinensis-orange1.1g015632m.g.gff3"
+  "Analysis", "Whole Genome Assembly and Annotation of Citrus sinensis"
+  "Organism", "Citrus sinensis"
+  "All other options", "leave as default"
+
+Finally, click the Import GFF3 file button. You'll notice a job was submitted to the jobs subsystem. Now, to complete the process we need the job to run. We'll do this manually:
+
+::
+
+  drush trp-run-jobs --username=administrator --root=/var/www/html
+
+You should see output similar to the following:
+
+::
+
+  Tripal Job Launcher
+  Running as user 'administrator'
+  -------------------
+  2018-06-29 18:00:50: There are 1 jobs queued.
+  2018-06-29 18:00:50: Job ID 8.
+  2018-06-29 18:00:50: Calling: tripal_run_importer(12)
+
+  Running 'Chado GFF3 File Loader' importer
+  NOTE: Loading of file is performed using a database transaction.
+  If it fails or is terminated prematurely then all insertions and
+  updates are rolled back and will not be found in the database
+
+  Opening /var/www/html/sites/default/files/tripal/users/1/Citrus_sinensis-orange1.1g015632m.g.gff3
+  Percent complete: 100.00%. Memory: 32,211,360 bytes.
+  Adding protein sequences if CDS exist and no proteins in GFF...
+  Setting ranks of children...
+
+  Done.
+
+  Remapping Chado Controlled vocabularies to Tripal Terms...
+  Done.
+
+.. note::
+
+  For very large GFF3 files the loader can take quite a while to complete.
+
+Loading FASTA files
+-------------------
+Using the Tripal GFF3 loader we were able to populate the database with the genomic features for our organism. However, those features now need nucleotide sequence data. To do this, we will load the nucleotide sequences for the mRNA features and the scaffold sequence. Navigate to the **Tripal → Data Loaders → Chado FASTA Loader**.
+
+.. image:: genomes_genes.2.png
+
+Before loading the FASTA file we must first know the Sequence Ontology (SO) term that describes the sequences we are about to upload. We can find the appropriate SO terms from our GFF file. In the GFF file we see the SO terms that correspond to our FASTA files are 'scaffold' and 'mRNA'.
+
+.. note::
+
+  It is important to ensure prior to importing, that the FASTA loader will be able to appropriately match the sequence in the FASTA file with existing sequences in the database. Before loading FASTA files, take special care to ensure the definition line of your FASTA file can uniquely identify the feature for the specific organism and sequence type.
+
+For example, in our GFF file an mRNA feature appears as follows:
+
+::
+
+  scaffold00001   phytozome6      mRNA    4058460 4062210 .       +       .       ID=PAC:18136217;Name=orange1.1g015632m;PACid=18136217;Parent=orange1.1g015632m.g
+
+Note that for this mRNA feature the ID is **PAC:18136217** and the name is **orange1.1g015632m**. In Chado, features always have a human readable name which does not need to be unique, and also a unique name which must be unique for the organism and SO type. In the GFF file, the ID becomes the unique name and the Name becomes the human readable name.
+
+In our FASTA file the definition line for this mRNA is:
+
+::
+
+  >orange1.1g015632m PAC:18136217 (mRNA) Citrus sinensis
+
+By default Tripal will match the sequence in a FASTA file with the feature that matches the first word in the definition line. In this case the first word is **orange1.1g015632m**. As defined in the GFF file, the name and unique name are different for this mRNA. However, we can see that the first word in the definition line of the FASTA file is the name and the second is the unique name. Therefore, when we load the FASTA file we should specify that we are matching by the name because it appears first in the definition line.
+
+If however, we cannot guarantee the that feature name is unique then we can use a regular expressions in the **Advanced Options** to tell Tripal where to find the name or unique name in the definition line of your FASTA file.
+
+.. note::
+
+  When loading FASTA files for features that have already been loaded via a GFF file, always choose "Update only" as the import method. Otherwise, Tripal may add the features in the FASTA file as new features if it cannot properly match them to existing features.
+
+Now, enter the following values in the fields on the web form:
+
+.. csv-table::
+  :header: "Field Name", "Value"
+
+  "FASTA file", "Upload the file named Citrus_sinensis-scaffold00001.fasta"
+  "Analysis", "Whole Genome Assembly and Annotation of Citrus sinensis"
+  "Organism", "Citrus sinensis (Sweet orange)"
+  "Sequence type", "supercontig (scaffold is an alias for supercontig in the sequence ontology)"
+  "Method", "Update only (we do not want to insert these are they should already be there)"
+  "Name Match Type", "Name"
+
+Click the Import Fasta File, and a job will be added to the jobs system. Run the job:
+
+::
+
+  drush trp-run-jobs --username=administrator --root=/var/www/html
+
+Notice that the loader reports the it "Found 1 sequences(s).". Next fill out the same form for the mRNA (transcripts) FASTA file:
+
+.. csv-table::
+  :header: "Field Name", "Value"
+
+  "FASTA file", "Upload the file named Citrus_sinensis-orange1.1g015632m.g.fasta"
+  "Analysis", "Whole Genome Assembly and Annotation of Citrus sinensis"
+  "Organism", "Citrus sinensis (Sweet orange)"
+  "Sequence type", "mRNA"
+  "Method", "Update only"
+  "Name Match", "Name"
+
+The FASTA loader has some advanced options. The advanced options allow you to create relationships between features and associate them with external databases. For example, the definition line for the mRNA in our FASTA file is:
+
+::
+
+  >orange1.1g015632m PAC:18136217 (mRNA) Citrus sinensis
+
+Here we have more information than just the feature name. We have a unique Phytozome accession number (e.g. PAC:18136217) for the mRNA. Using the **External Database Reference** section under **Additional Options** we can import this information to associate the Phytozome accession with the features.  A regular expression is required to uniquely capture that ID.  In the example above the unique accession is 18136217.  Because Tripal is a PHP application, the syntax for regular expressions follows the PHP method. Documentation for regular expressions used in PHP can be found `here <http://php.net/manual/en/reference.pcre.pattern.syntax.php>`_.  Enter the following value to make the associate between the mRNA and it's corresponding accession at Phytozome:
+
+.. csv-table::
+  :header: "Field Name", "Value"
+
+  "External Database", "Phytozome"
+  "Regular expression for the accession", "^.*PAC:(\d+).*$"
+
+Remember, we have the name **Phytozome** in our **External Database** drop down because we manually added it as a database cross reference earlier in the turorial.  After adding the values above, click the **Import FASTA file** button, and manually run the submitted job:
+
+::
+
+  drush trp-run-jobs --username=administrator --root=/var/www/html
+
+Now the scaffold sequence and mRNA sequences are loaded!
+
+.. note:
+
+  If the name of the gene to which this mRNA belonged was also on the definition line, we could use the **Relationships** section in the **Advanced Options** to link this mRNA with it's gene parent. Fortunately, this information is also in our GFF file and these relationships have already been made.
+
+.. note::
+
+  It is not required to load the mRNA sequences as those can be derived from their alignments with the scaffold sequence. However, in Chado the **feature** table has a **residues** column. Therefore, it is best practice to load the sequence when possible.
+
+Creating Gene Pages
+----------------------
+Now that we've loaded our feature data, we must publish them. This is different than when we manually created our Organism and Analysis pages.  Using the GFF and FASTA loaders we imported our data into Chado, but currently there are no published pages for this data that we loaded.  To publish these genomic features, navigating to Structure → Tripal Content Types and click the link titled Publish Chado Content.  The following page appears:
+
+.. image:: genomes_genes.3.png
+
+Here we can specify the types of content to publish. For our site we want to offer both gene and mRNA pages (these types were present in our GFF file). First, to create pages for genes select 'Gene' from the dropdown.  A new Filter section is present and when opened appears as follows.
+
+.. image:: genomes_genes.4.png
+
+The **Filters** section allows you to provide filters to limit what you want to publish.  For example, if you only want to publish genes for a single organism you can select that organism in the Organism drop down list.  We only have one organism in our site, but for the sake of experience, add a filter to publish only genes for Citrus sinesis by selecting it from the Organism drop down.  Scroll to the bottom a click the Publish button.  A new job is added to the job queue.  Manually run the job:
+
+::
+
+  drush trp-run-jobs --username=administrator --root=/var/www/html
+
+You should see output similar to the following:
+
+::
+
+  Tripal Job Launcher
+  Running as user 'administrator'
+  -------------------
+  Calling: tripal_chado_publish_records(Array, 12)
+
+  NOTE: publishing records is performed using a database transaction.
+  If the load fails or is terminated prematurely then the entire set of
+  is rolled back with no changes to the database
+
+  Succesfully published 1 Gene record(s).
+
+Here we see that 1 gene was successfully published. This is because the GFF file we used previously to import the genes only had one gene present.
+
+Now, repeat the steps above to publish the mRNA content type.  You should see that 9 mRNA records were published:
+
+::
+
+  Tripal Job Launcher
+  Running as user 'administrator'
+  -------------------
+  Calling: tripal_chado_publish_records(Array, 13)
+
+  NOTE: publishing records is performed using a database transaction.
+  If the load fails or is terminated prematurely then the entire set of
+  is rolled back with no changes to the database
+
+  Succesfully published 9 mRNA record(s).
+
+.. note::
+
+  It is not necessary to publish all types of features in the GFF file. For example, we do not want to publish features of type **scaffold**. The feature is large and would have many relationships to other features, as well as a very long nucleotide sequence.  These can greatly slow down page loading, and in general would be overwhelming to the user to view on one page. As another example, each **mRNA** is composed of several **CDS** features. These **CDS** features do not need their own page and therefore do not need to be published.
+
+Now, we can view our gene and mRNA pages. Click the Find Tripal Content link. Find and click the new page titled **orange1.1g015632m.g**. Here we can see the gene feature we added and its corresponding mRNA's.
+
+.. image:: genomes_genes.5.png
+
+Next find an mRNA page to view.  Remember when we loaded our FASTA file for mRNA that we associated the record with Phytozome.  On these mRNA pages you will see a link in the left side bar titled **Database Cross Reference**.  Clicking that will open a panel with a link to Phytozome.  This link appears because:
+
+- We added a Database Cross Reference for Phytozome in a previous step
+- We associated the Phytozome accession with the features using a regular expression when importing the FASTA file.
+
+All data that appears on the page is derived from the GFF file and the FASTA  files we loaded.

docs/user_guide/example_genomics/organisms.rst

@@ -0,0 +1,93 @@
+Organisms
+=========
+Before we can load our data we must first have an organism to which the data will be associated.  Chado v1.3 does not come preloaded with any organisms (although previous version of Chado do).  For this tutorial we will import genomic data for Citrus sinesis (sweet orange), so we must first create the organism.
+
+Creating an Organism Page
+-------------------------
+We can  add the organism using the **Add Tripal Content** link in the top administrative menu. The **Add Tripal Content** page has several content types already available, including the **Organism** content type.
+
+.. note::
+
+  Drupal provides it's own content types such as **Article** and **Basic Page**.  These content types are referred to as **nodes** in Drupal speak. You can add these content types via the **Add Content** page. Tripal v3 derived content types are separated from these Drupal content types.
+
+.. image:: organisms.1.png
+
+To add a new organism  click the **Organism** link and a form will appear with multiple fields. Fill in the fields with these values:
+
+.. csv-table::
+  :header: "Field Name", "Value"
+
+  "Abbreviation", "C. sinensis"
+  "Genus", "Citrus"
+  "Species", "sinensis"
+  "Common name", "Sweet orange"
+  "Description",	"Sweet orange is the No.1 citrus production in the world, accounting for about 70% of the total. Brazil, Flordia (USA), and China are the three largest sweet orange producers. Sweet orange fruits have very tight peel and are classified into the hard-to-peel group. They are often used for juice processing, rather than fresh consumption. Valencia, Navel, Blood, Acidless, and other subtypes are bud mutants of common sweet orange varieties. Sweet orange is considered as an introgression of a natural hybrid of mandarin and pummelo; some estimates shows more mandarin genomic background than pummelo. The genome size is estimated at 380Mb across 9 haploid chromosomes."
+  "Image", .. image:: organisms.citrus_sinensis.jpg
+
+Leave all remaining fields empty and save the page.  You should now have an organism page that appears as follows:
+
+.. image:: organisms.2.png
+
+.. note::
+
+  The layout of the organism page is provided by the **tripal_ds** module that was enabled during Tripal installation.  If you decided not to enable that module then your page will look quite different.
+
+This page has three primary sections.  A left sidebar that contains a search box and a **block** titled **Navigation**.  To the right of the sidebar is the content section.  Here the content is divided into two columns.  In the first column is a table of contents listing the "Table of Contents" of the page.  Currently this list simply contains the **Summary**. The second column contains all of the content about the organism.  Clicking the links in the table of contents causes each section to slide to the top of the page for viewing while all other content slides downward.  Users need not scroll the entire page to see all content.  The close link (X) at the top right of each section allows the user to remove sections they may not want to see and which clutter their view.
+
+There is more information about an organism that does not appear on this page.   By default, Tripal will not show fields that have no data.  However, if you prefer, you can change this behavior by configuring the content type to not hide empty fields.  You can find this option by navigating to **Structure > Tripal Content Types**, and click on the **edit** link for the content type. You will find the option to toggle this behavior there.
+
+If you do not like this layout you can change it!  One of the benefits for using Drupal is the ability to customize the entire look and feel of your site.  Tripal v3 along with Drupal allow you to reorganize this (and any) page however you like.  This default layout is provided by the **tripal_ds** module to help simplify the process of laying out a page.  If you decided not to enable the **tripal_ds** module then your page will require manual layout. Later in this tutorial instructions to reorganize and re-theme the Tripal content pages are provided.  No programming is required to do this.
+
+Load data from NCBI Taxonomy
+----------------------------
+Tripal makes it easy to import additional information about any organisms within a Tripal site from the `NCBI Taxonomy database <https://www.ncbi.nlm.nih.gov/taxonomy>`_.  The importer will only import data for species that you currently have in the Tripal database.  The taxonomic names must match those in the NCBI Taxonomy database.  Currently, we only have a single organism (Citrus sinensis) and we will import additional properties for this organism from NCBI but we can return later to import data for new organisms we may add later.  To import additional organism details, navigate to **Tripal → Data Loaders → Chado NCBI Taxonomy Loader**.  The following page appears:
+
+.. image:: organisms.taxonomy_loader.png
+
+Click the checbox beside the 'Import taxonomy for existing species' and click Submit.  Now run the submitted job:
+
+::
+
+  drush trp-run-jobs --username=administrator --root=/var/www/html
+
+You will see the following output:
+
+::
+
+  Tripal Job Launcher (in parallel)
+  Running as user 'administrator'
+  -------------------
+  2017-10-06 15:45:47: There are 1 jobs queued.
+  2017-10-06 15:45:47: Calling: tripal_chado_ncbi_taxonomy_import()
+
+  NOTE: Importing of NCBI taxonomy data is performed using a database transaction.
+  If the load fails or is terminated prematurely then the entire set of
+  insertions/updates is rolled back and will not be found in the database
+
+  2711    Citrus sinensis
+
+Adding New Fields
+-----------------
+
+We have now imported many new properties about the Citrus sinensis organism from NCBI Taxonomy.  However, these properties won't show up on the page automatically.  We need to tell Drupal that our organism pages now have new property fields for display.   To do this, navigate to **Structure → Tripal Content Types** and in the row for the Organism content type, click the link titled managed fields.   Here we see a list the fields that associated with an Organism content type.  Click the link at the top of the page **Check for new fields**.  You will see that several new fields have been added.
+
+.. image:: organisms.new_fields1.png
+
+Drupal now knows about these new fields!  But if we were to look at the Citrus sinensis page we would see that the new properties do not appear.  Despite that Drupal knows about the fields it has disabled their display.  To enable display of these fields click the **Manage Display** tab at the top right of the page.  Here all of the fields are organized into the structure that they will be displayed on the page.   Later in this tutorial a more formal description is provided about how you use this interface to change the way the page appears.  For now, we simply need to get the new fields to be shown.    Scroll to the bottom of the page and the new fields can be seen in the **Disabled** section.
+
+.. image:: organisms.new_fields2.png
+
+We can move these new fields out of the Disabled section by clicking on the cross-hair icons to the left of the name and dragging the field into a section above.   Drag these fields into the **Summary** section underneath the **Summary Table**.  Notice in the screenshot below that the fields that were once in the **Disabled** section are now in the **Summary Table** section.  Click the **Save** button at the bottom to make all changes final.
+
+.. image:: organisms.new_fields3.png
+
+Now, if we return to the organism page we will see these new properties were added to the page inside of the Summary Table.
+
+.. image:: organisms.updated_page1.png
+
+Further Customizations
+----------------------
+
+You may not like this arrangement of fields.  You may prefer to place these extra fields inside of a new **pane** rather than inside of the **Summary pane**.  Perhaps a pane named Additional Details.  You can rearrange the order of these fields and create new panes, as desired by following the more details instructions on the `Configure Page Display page <../configuring_page_display>`_ of this tutorial.  For example, the following shows these fields organized into a new pane named **Additional Details** which is separate from the **Summary** Pane.  Note the table of contents sidebar now lists the **Summary** and **Additional Details** links.  When clicked, the pane selected by the user migrates to the top of the page
+
+.. image:: organisms.updated_page2.png