# Copyright (C) 2013 by Yanbo Ye (yeyanbo289@gmail.com) # # This file is part of the Biopython distribution and governed by your # choice of the "Biopython License Agreement" or the "BSD 3-Clause License". # Please see the LICENSE file that should have been included as part of this # package. """Classes and methods for finding consensus trees. This module contains a ``_BitString`` class to assist the consensus tree searching and some common consensus algorithms such as strict, majority rule and adam consensus. """ import itertools import random from ast import literal_eval from Bio.Align import MultipleSeqAlignment from Bio.Phylo import BaseTree class _BitString(str): """Helper class for binary string data (PRIVATE). Assistant class of binary string data used for storing and counting compatible clades in consensus tree searching. It includes some binary manipulation(&|^~) methods. _BitString is a sub-class of ``str`` object that only accepts two characters('0' and '1'), with additional functions for binary-like manipulation(&|^~). It is used to count and store the clades in multiple trees in consensus tree searching. During counting, the clades will be considered the same if their terminals(in terms of ``name`` attribute) are the same. For example, let's say two trees are provided as below to search their strict consensus tree:: tree1: (((A, B), C),(D, E)) tree2: ((A, (B, C)),(D, E)) For both trees, a _BitString object '11111' will represent their root clade. Each '1' stands for the terminal clade in the list [A, B, C, D, E](the order might not be the same, it's determined by the ``get_terminal`` method of the first tree provided). For the clade ((A, B), C) in tree1 and (A, (B, C)) in tree2, they both can be represented by '11100'. Similarly, '11000' represents clade (A, B) in tree1, '01100' represents clade (B, C) in tree2, and '00011' represents clade (D, E) in both trees. So, with the ``_count_clades`` function in this module, finally we can get the clade counts and their _BitString representation as follows (the root and terminals are omitted):: clade _BitString count ABC '11100' 2 DE '00011' 2 AB '11000' 1 BC '01100' 1 To get the _BitString representation of a clade, we can use the following code snippet:: # suppose we are provided with a tree list, the first thing to do is # to get all the terminal names in the first tree term_names = [term.name for term in trees[0].get_terminals()] # for a specific clade in any of the tree, also get its terminal names clade_term_names = [term.name for term in clade.get_terminals()] # then create a boolean list boolvals = [name in clade_term_names for name in term_names] # create the string version and pass it to _BitString bitstr = _BitString(''.join(map(str, map(int, boolvals)))) # or, equivalently: bitstr = _BitString.from_bool(boolvals) To convert back:: # get all the terminal clades of the first tree terms = [term for term in trees[0].get_terminals()] # get the index of terminal clades in bitstr index_list = bitstr.index_one() # get all terminal clades by index clade_terms = [terms[i] for i in index_list] # create a new calde and append all the terminal clades new_clade = BaseTree.Clade() new_clade.clades.extend(clade_terms) Examples -------- >>> from Bio.Phylo.Consensus import _BitString >>> bitstr1 = _BitString('11111') >>> bitstr2 = _BitString('11100') >>> bitstr3 = _BitString('01101') >>> bitstr1 _BitString('11111') >>> bitstr2 & bitstr3 _BitString('01100') >>> bitstr2 | bitstr3 _BitString('11101') >>> bitstr2 ^ bitstr3 _BitString('10001') >>> bitstr2.index_one() [0, 1, 2] >>> bitstr3.index_one() [1, 2, 4] >>> bitstr3.index_zero() [0, 3] >>> bitstr1.contains(bitstr2) True >>> bitstr2.contains(bitstr3) False >>> bitstr2.independent(bitstr3) False >>> bitstr1.iscompatible(bitstr2) True >>> bitstr2.iscompatible(bitstr3) False """ def __new__(cls, strdata): """Init from a binary string data.""" if isinstance(strdata, str) and len(strdata) == strdata.count( "0" ) + strdata.count("1"): return str.__new__(cls, strdata) else: raise TypeError( "The input should be a binary string composed of '0' and '1'" ) def __and__(self, other): selfint = literal_eval("0b" + self) otherint = literal_eval("0b" + other) resultint = selfint & otherint return _BitString(bin(resultint)[2:].zfill(len(self))) def __or__(self, other): selfint = literal_eval("0b" + self) otherint = literal_eval("0b" + other) resultint = selfint | otherint return _BitString(bin(resultint)[2:].zfill(len(self))) def __xor__(self, other): selfint = literal_eval("0b" + self) otherint = literal_eval("0b" + other) resultint = selfint ^ otherint return _BitString(bin(resultint)[2:].zfill(len(self))) def __rand__(self, other): selfint = literal_eval("0b" + self) otherint = literal_eval("0b" + other) resultint = otherint & selfint return _BitString(bin(resultint)[2:].zfill(len(self))) def __ror__(self, other): selfint = literal_eval("0b" + self) otherint = literal_eval("0b" + other) resultint = otherint | selfint return _BitString(bin(resultint)[2:].zfill(len(self))) def __rxor__(self, other): selfint = literal_eval("0b" + self) otherint = literal_eval("0b" + other) resultint = otherint ^ selfint return _BitString(bin(resultint)[2:].zfill(len(self))) def __repr__(self): return "_BitString(" + str.__repr__(self) + ")" def index_one(self): """Return a list of positions where the element is '1'.""" return [i for i, n in enumerate(self) if n == "1"] def index_zero(self): """Return a list of positions where the element is '0'.""" return [i for i, n in enumerate(self) if n == "0"] def contains(self, other): """Check if current bitstr1 contains another one bitstr2. That is to say, the bitstr2.index_one() is a subset of bitstr1.index_one(). Examples: "011011" contains "011000", "011001", "000011" Be careful, "011011" also contains "000000". Actually, all _BitString objects contain all-zero _BitString of the same length. """ xorbit = self ^ other return xorbit.count("1") == self.count("1") - other.count("1") def independent(self, other): """Check if current bitstr1 is independent of another one bitstr2. That is to say the bitstr1.index_one() and bitstr2.index_one() have no intersection. Be careful, all _BitString objects are independent of all-zero _BitString of the same length. """ xorbit = self ^ other return xorbit.count("1") == self.count("1") + other.count("1") def iscompatible(self, other): """Check if current bitstr1 is compatible with another bitstr2. Two conditions are considered as compatible: 1. bitstr1.contain(bitstr2) or vice versa; 2. bitstr1.independent(bitstr2). """ return self.contains(other) or other.contains(self) or self.independent(other) @classmethod def from_bool(cls, bools): return cls("".join(map(str, map(int, bools)))) def strict_consensus(trees): """Search strict consensus tree from multiple trees. :Parameters: trees : iterable iterable of trees to produce consensus tree. """ trees_iter = iter(trees) first_tree = next(trees_iter) terms = first_tree.get_terminals() bitstr_counts, tree_count = _count_clades(itertools.chain([first_tree], trees_iter)) # Store bitstrs for strict clades strict_bitstrs = [ bitstr for bitstr, t in bitstr_counts.items() if t[0] == tree_count ] strict_bitstrs.sort(key=lambda bitstr: bitstr.count("1"), reverse=True) # Create root root = BaseTree.Clade() if strict_bitstrs[0].count("1") == len(terms): root.clades.extend(terms) else: raise ValueError("Taxons in provided trees should be consistent") # make a bitstr to clades dict and store root clade bitstr_clades = {strict_bitstrs[0]: root} # create inner clades for bitstr in strict_bitstrs[1:]: clade_terms = [terms[i] for i in bitstr.index_one()] clade = BaseTree.Clade() clade.clades.extend(clade_terms) for bs, c in bitstr_clades.items(): # check if it should be the parent of current clade if bs.contains(bitstr): # remove old bitstring del bitstr_clades[bs] # update clade children c.clades = [child for child in c.clades if child not in clade_terms] # set current clade as child of c c.clades.append(clade) # update bitstring bs = bs ^ bitstr # update clade bitstr_clades[bs] = c break # put new clade bitstr_clades[bitstr] = clade return BaseTree.Tree(root=root) def majority_consensus(trees, cutoff=0): """Search majority rule consensus tree from multiple trees. This is a extend majority rule method, which means the you can set any cutoff between 0 ~ 1 instead of 0.5. The default value of cutoff is 0 to create a relaxed binary consensus tree in any condition (as long as one of the provided trees is a binary tree). The branch length of each consensus clade in the result consensus tree is the average length of all counts for that clade. :Parameters: trees : iterable iterable of trees to produce consensus tree. """ tree_iter = iter(trees) first_tree = next(tree_iter) terms = first_tree.get_terminals() bitstr_counts, tree_count = _count_clades(itertools.chain([first_tree], tree_iter)) # Sort bitstrs by descending #occurrences, then #tips, then tip order bitstrs = sorted( bitstr_counts.keys(), key=lambda bitstr: (bitstr_counts[bitstr][0], bitstr.count("1"), str(bitstr)), reverse=True, ) root = BaseTree.Clade() if bitstrs[0].count("1") == len(terms): root.clades.extend(terms) else: raise ValueError("Taxons in provided trees should be consistent") # Make a bitstr-to-clades dict and store root clade bitstr_clades = {bitstrs[0]: root} # create inner clades for bitstr in bitstrs[1:]: # apply majority rule count_in_trees, branch_length_sum = bitstr_counts[bitstr] confidence = 100.0 * count_in_trees / tree_count if confidence < cutoff * 100.0: break clade_terms = [terms[i] for i in bitstr.index_one()] clade = BaseTree.Clade() clade.clades.extend(clade_terms) clade.confidence = confidence clade.branch_length = branch_length_sum / count_in_trees bsckeys = sorted(bitstr_clades, key=lambda bs: bs.count("1"), reverse=True) # check if current clade is compatible with previous clades and # record its possible parent and child clades. compatible = True parent_bitstr = None child_bitstrs = [] # multiple independent children for bs in bsckeys: if not bs.iscompatible(bitstr): compatible = False break # assign the closest ancestor as its parent # as bsckeys is sorted, it should be the last one if bs.contains(bitstr): parent_bitstr = bs # assign the closest descendant as its child # the largest and independent clades if ( bitstr.contains(bs) and bs != bitstr and all(c.independent(bs) for c in child_bitstrs) ): child_bitstrs.append(bs) if not compatible: continue if parent_bitstr: # insert current clade; remove old bitstring parent_clade = bitstr_clades.pop(parent_bitstr) # update parent clade children parent_clade.clades = [ c for c in parent_clade.clades if c not in clade_terms ] # set current clade as child of parent_clade parent_clade.clades.append(clade) # update bitstring # parent = parent ^ bitstr # update clade bitstr_clades[parent_bitstr] = parent_clade if child_bitstrs: remove_list = [] for c in child_bitstrs: remove_list.extend(c.index_one()) child_clade = bitstr_clades[c] parent_clade.clades.remove(child_clade) clade.clades.append(child_clade) remove_terms = [terms[i] for i in remove_list] clade.clades = [c for c in clade.clades if c not in remove_terms] # put new clade bitstr_clades[bitstr] = clade if (len(bitstr_clades) == len(terms) - 1) or ( len(bitstr_clades) == len(terms) - 2 and len(root.clades) == 3 ): break return BaseTree.Tree(root=root) def adam_consensus(trees): """Search Adam Consensus tree from multiple trees. :Parameters: trees : list list of trees to produce consensus tree. """ clades = [tree.root for tree in trees] return BaseTree.Tree(root=_part(clades), rooted=True) def _part(clades): """Recursive function for Adam Consensus algorithm (PRIVATE).""" new_clade = None terms = clades[0].get_terminals() term_names = [term.name for term in terms] if len(terms) == 1 or len(terms) == 2: new_clade = clades[0] else: bitstrs = {_BitString("1" * len(terms))} for clade in clades: for child in clade.clades: bitstr = _clade_to_bitstr(child, term_names) to_remove = set() to_add = set() for bs in bitstrs: if bs == bitstr: continue elif bs.contains(bitstr): to_add.add(bitstr) to_add.add(bs ^ bitstr) to_remove.add(bs) elif bitstr.contains(bs): to_add.add(bs ^ bitstr) elif not bs.independent(bitstr): to_add.add(bs & bitstr) to_add.add(bs & bitstr ^ bitstr) to_add.add(bs & bitstr ^ bs) to_remove.add(bs) # bitstrs = bitstrs | to_add bitstrs ^= to_remove if to_add: for ta in sorted(to_add, key=lambda bs: bs.count("1")): independent = True for bs in bitstrs: if not ta.independent(bs): independent = False break if independent: bitstrs.add(ta) new_clade = BaseTree.Clade() for bitstr in sorted(bitstrs): indices = bitstr.index_one() if len(indices) == 1: new_clade.clades.append(terms[indices[0]]) elif len(indices) == 2: bifur_clade = BaseTree.Clade() bifur_clade.clades.append(terms[indices[0]]) bifur_clade.clades.append(terms[indices[1]]) new_clade.clades.append(bifur_clade) elif len(indices) > 2: part_names = [term_names[i] for i in indices] next_clades = [] for clade in clades: next_clades.append(_sub_clade(clade, part_names)) # next_clades = [clade.common_ancestor([clade.find_any(name=name) for name in part_names]) for clade in clades] new_clade.clades.append(_part(next_clades)) return new_clade def _sub_clade(clade, term_names): """Extract a compatible subclade that only contains the given terminal names (PRIVATE).""" term_clades = [clade.find_any(name) for name in term_names] sub_clade = clade.common_ancestor(term_clades) if len(term_names) != sub_clade.count_terminals(): temp_clade = BaseTree.Clade() temp_clade.clades.extend(term_clades) for c in sub_clade.find_clades(terminal=False, order="preorder"): if c == sub_clade.root: continue children = set(c.find_clades(terminal=True)) & set(term_clades) if children: for tc in temp_clade.find_clades(terminal=False, order="preorder"): tc_children = set(tc.clades) tc_new_clades = tc_children - children if children.issubset(tc_children) and tc_new_clades: tc.clades = list(tc_new_clades) child_clade = BaseTree.Clade() child_clade.clades.extend(list(children)) tc.clades.append(child_clade) sub_clade = temp_clade return sub_clade def _count_clades(trees): """Count distinct clades (different sets of terminal names) in the trees (PRIVATE). Return a tuple first a dict of bitstring (representing clade) and a tuple of its count of occurrences and sum of branch length for that clade, second the number of trees processed. :Parameters: trees : iterable An iterable that returns the trees to count """ bitstrs = {} tree_count = 0 for tree in trees: tree_count += 1 clade_bitstrs = _tree_to_bitstrs(tree) for clade in tree.find_clades(terminal=False): bitstr = clade_bitstrs[clade] if bitstr in bitstrs: count, sum_bl = bitstrs[bitstr] count += 1 sum_bl += clade.branch_length or 0 bitstrs[bitstr] = (count, sum_bl) else: bitstrs[bitstr] = (1, clade.branch_length or 0) return bitstrs, tree_count def get_support(target_tree, trees, len_trees=None): """Calculate branch support for a target tree given bootstrap replicate trees. :Parameters: target_tree : Tree tree to calculate branch support for. trees : iterable iterable of trees used to calculate branch support. len_trees : int optional count of replicates in trees. len_trees must be provided when len(trees) is not a valid operation. """ term_names = sorted(term.name for term in target_tree.find_clades(terminal=True)) bitstrs = {} size = len_trees if size is None: try: size = len(trees) except TypeError: raise TypeError( "Trees does not support len(trees), " "you must provide the number of replicates in trees " "as the optional parameter len_trees." ) from None for clade in target_tree.find_clades(terminal=False): bitstr = _clade_to_bitstr(clade, term_names) bitstrs[bitstr] = (clade, 0) for tree in trees: for clade in tree.find_clades(terminal=False): bitstr = _clade_to_bitstr(clade, term_names) if bitstr in bitstrs: c, t = bitstrs[bitstr] c.confidence = (t + 1) * 100.0 / size bitstrs[bitstr] = (c, t + 1) return target_tree def bootstrap(msa, times): """Generate bootstrap replicates from a multiple sequence alignment (OBSOLETE). :Parameters: msa : MultipleSeqAlignment multiple sequence alignment to generate replicates. times : int number of bootstrap times. """ length = len(msa[0]) i = 0 while i < times: i += 1 item = None for j in range(length): col = random.randint(0, length - 1) if not item: item = msa[:, col : col + 1] else: item += msa[:, col : col + 1] yield item def bootstrap_trees(alignment, times, tree_constructor): """Generate bootstrap replicate trees from a multiple sequence alignment. :Parameters: alignment : Alignment or MultipleSeqAlignment object multiple sequence alignment to generate replicates. times : int number of bootstrap times. tree_constructor : TreeConstructor tree constructor to be used to build trees. """ if isinstance(alignment, MultipleSeqAlignment): length = len(alignment[0]) for i in range(times): bootstrapped_alignment = None for j in range(length): col = random.randint(0, length - 1) if bootstrapped_alignment is None: bootstrapped_alignment = alignment[:, col : col + 1] else: bootstrapped_alignment += alignment[:, col : col + 1] tree = tree_constructor.build_tree(bootstrapped_alignment) yield tree else: n, m = alignment.shape for i in range(times): cols = [random.randint(0, m - 1) for j in range(m)] tree = tree_constructor.build_tree(alignment[:, cols]) yield tree def bootstrap_consensus(alignment, times, tree_constructor, consensus): """Consensus tree of a series of bootstrap trees for a multiple sequence alignment. :Parameters: alignment : Alignment or MultipleSeqAlignment object Multiple sequence alignment to generate replicates. times : int Number of bootstrap times. tree_constructor : TreeConstructor Tree constructor to be used to build trees. consensus : function Consensus method in this module: ``strict_consensus``, ``majority_consensus``, ``adam_consensus``. """ trees = bootstrap_trees(alignment, times, tree_constructor) tree = consensus(trees) return tree def _clade_to_bitstr(clade, tree_term_names): """Create a BitString representing a clade, given ordered tree taxon names (PRIVATE).""" clade_term_names = {term.name for term in clade.find_clades(terminal=True)} return _BitString.from_bool((name in clade_term_names) for name in tree_term_names) def _tree_to_bitstrs(tree): """Create a dict of a tree's clades to corresponding BitStrings (PRIVATE).""" clades_bitstrs = {} term_names = [term.name for term in tree.find_clades(terminal=True)] for clade in tree.find_clades(terminal=False): bitstr = _clade_to_bitstr(clade, term_names) clades_bitstrs[clade] = bitstr return clades_bitstrs def _bitstring_topology(tree): """Generate a branch length dict for a tree, keyed by BitStrings (PRIVATE). Create a dict of all clades' BitStrings to the corresponding branch lengths (rounded to 5 decimal places). """ bitstrs = {} for clade, bitstr in _tree_to_bitstrs(tree).items(): bitstrs[bitstr] = round(clade.branch_length or 0.0, 5) return bitstrs def _equal_topology(tree1, tree2): """Are two trees are equal in terms of topology and branch lengths (PRIVATE). (Branch lengths checked to 5 decimal places.) """ term_names1 = {term.name for term in tree1.find_clades(terminal=True)} term_names2 = {term.name for term in tree2.find_clades(terminal=True)} return (term_names1 == term_names2) and ( _bitstring_topology(tree1) == _bitstring_topology(tree2) )