From 37848a616882d3f4958d94c7a13836a376f88e81 Mon Sep 17 00:00:00 2001 From: Zia Truong <194475824+electricEpilith@users.noreply.github.com> Date: Wed, 12 Nov 2025 14:32:58 -0800 Subject: [PATCH 01/77] some progress on hub label integration? --- src/snarl_distance_index.cpp | 702 +++++++++++++++++++---------------- 1 file changed, 385 insertions(+), 317 deletions(-) diff --git a/src/snarl_distance_index.cpp b/src/snarl_distance_index.cpp index 9821a8ebe8..c6384a9f2b 100644 --- a/src/snarl_distance_index.cpp +++ b/src/snarl_distance_index.cpp @@ -781,7 +781,24 @@ SnarlDistanceIndex::TemporaryDistanceIndex make_temporary_distance_index( return temp_index; } - +static void populate_distance_matrix_row(SnarlDistanceIndex::TemporaryDistanceIndex& temp_index, pair& snarl_index, SnarlDistanceIndex::TemporaryDistanceIndex::TemporarySnarlRecord& temp_snarl_record, const pair& start_index, const HandleGraph* graph, size_t start_rank, bool is_internal_node, size_t size_limit); + +/* +Fills in required distance matrix rows for each child +- Normal snarl: all rows +- Oversized snarl: boundaries and tips +- size_limit == 0: no distances in index, so no rows +- Top-level chain distances only: ??? +*/ +static void populate_distance_matrix_if_needed(SnarlDistanceIndex::TemporaryDistanceIndex& temp_index, pair& snarl_index, SnarlDistanceIndex::TemporaryDistanceIndex::TemporarySnarlRecord& temp_snarl_record, vector>& all_children, const HandleGraph* graph, size_t size_limit); + +/* +Does three things: +- Builds temp graph that hub labels will be built on +- Builds the hub labels +- Stores labels in temp_snarl_record +*/ +static void populate_hub_labeling(SnarlDistanceIndex::TemporaryDistanceIndex& temp_index, pair& snarl_index, SnarlDistanceIndex::TemporaryDistanceIndex::TemporarySnarlRecord& temp_snarl_record, vector>& all_children, const HandleGraph* graph); /*Fill in the snarl index. * The index will already know its boundaries and everything knows their relationships in the @@ -1098,17 +1115,6 @@ void populate_snarl_index( * Start a dijkstra traversal from each node side in the snarl and record all distances */ - - if (size_limit != 0 && !only_top_level_chain_distances) { - //If we are saving distances - //Reserve enough space to store all possible distances - temp_snarl_record.distances.reserve( temp_snarl_record.node_count > size_limit - ? temp_snarl_record.node_count * 2 - : temp_snarl_record.node_count * temp_snarl_record.node_count); - } else { - temp_snarl_record.include_distances = false; - } - if (size_limit != 0 && temp_snarl_record.node_count > size_limit) { temp_index.use_oversized_snarls = true; } @@ -1119,6 +1125,68 @@ void populate_snarl_index( all_children.emplace_back(SnarlDistanceIndex::TEMP_NODE, temp_snarl_record.end_node_id); } + if (size_limit == 0) { + temp_snarl_record.include_distances = false; + } else if (only_top_level_chain_distances) { + temp_snarl_record.include_distances = false; + } else if (temp_index.use_oversized_snarls) { + populate_hub_labeling(temp_index, snarl_index, temp_snarl_record, all_children, graph); + } else { + populate_distance_matrix_if_needed(temp_index, snarl_index, temp_snarl_record, all_children, graph, size_limit); + } + + //If this is a simple snarl (one with only single nodes that connect to the start and end nodes), then + // we want to remember if the child nodes are reversed + if (temp_snarl_record.is_simple) { + for (size_t i = 0 ; i < temp_snarl_record.node_count ; i++) { + //Get the index of the child + const pair& child_index = temp_snarl_record.children[i]; + //Which is a node +#ifdef debug_distance_indexing + assert(child_index.first == SnarlDistanceIndex::TEMP_NODE); +#endif + + //And get the record + SnarlDistanceIndex::TemporaryDistanceIndex::TemporaryNodeRecord& temp_node_record = + temp_index.temp_node_records[child_index.second-temp_index.min_node_id]; + size_t rank =temp_node_record.rank_in_parent; + + + + //Set the orientation of this node in the simple snarl + temp_node_record.reversed_in_parent = temp_node_record.distance_left_start == std::numeric_limits::max(); + + } + } + + //Now that the distances are filled in, predict the size of the snarl in the index + temp_index.max_index_size += temp_snarl_record.get_max_record_length(); + if (temp_snarl_record.is_simple) { + temp_index.max_index_size -= (temp_snarl_record.children.size() * SnarlDistanceIndex::TemporaryDistanceIndex::TemporaryNodeRecord::get_max_record_length()); + } + + // For simple snarl records, need 11 + 11 + number of bits for the number of children + temp_index.max_bits = std::max(temp_index.max_bits, 22 + SnarlDistanceIndex::bit_width(temp_snarl_record.children.size())); +} + +void populate_hub_labeling(SnarlDistanceIndex::TemporaryDistanceIndex& temp_index, pair& snarl_index, SnarlDistanceIndex::TemporaryDistanceIndex::TemporarySnarlRecord& temp_snarl_record, vector>& all_children, const HandleGraph* graph) { + CHOverlay ov = make_boost_graph(temp_index, snarl_index, temp_snarl_record, all_children, graph); + make_contraction_hierarchy(ov); + + vector> labels; labels.resize(num_vertices(ov)); + vector> labels_rev; labels_rev.resize(num_vertices(ov)); + create_labels(labels, labels_rev, ov); + //TODO: Put labels in temp_snarl_record +} + +void populate_distance_matrix_if_needed(SnarlDistanceIndex::TemporaryDistanceIndex& temp_index, pair& snarl_index, SnarlDistanceIndex::TemporaryDistanceIndex::TemporarySnarlRecord& temp_snarl_record, vector>& all_children, const HandleGraph* graph, size_t size_limit) { + if (size_limit != 0 && !only_top_level_chain_distances) { + //If we are saving distances + //Reserve enough space to store all possible distances + temp_snarl_record.distances.reserve( temp_snarl_record.node_count > size_limit + ? temp_snarl_record.node_count * 2 + : temp_snarl_record.node_count * temp_snarl_record.node_count); + } while (!all_children.empty()) { const pair start_index = std::move(all_children.back()); all_children.pop_back(); @@ -1176,357 +1244,357 @@ void populate_snarl_index( } //TODO: //else { // assert(start_rank != 0 && start_rank != 1); - //} + //} - if ( (temp_snarl_record.node_count > size_limit || size_limit == 0 || only_top_level_chain_distances) && (temp_snarl_record.is_root_snarl || (!start_is_tip && - start_rank != 0 && start_rank != 1))) { + //traversal start is not a tip or a boundary node + bool start_normal_child = (!start_is_tip && start_rank != 0 && start_rank != 1); + + if ( (temp_snarl_record.node_count > size_limit || size_limit == 0 || only_top_level_chain_distances) && (temp_snarl_record.is_root_snarl || start_normal_child)) { //If we don't care about internal distances, and we also are not at a boundary or tip //TODO: Why do we care about tips specifically? continue; } + //getting here means snarl is not oversized + //fill in all distances for a row + populate_distance_matrix_row(temp_index, snarl_index, temp_snarl_record, start_index, graph, start_rank, is_internal_node, size_limit); + } +} + + + +void populate_distance_matrix_row(SnarlDistanceIndex::TemporaryDistanceIndex& temp_index, pair& snarl_index, SnarlDistanceIndex::TemporaryDistanceIndex::TemporarySnarlRecord& temp_snarl_record, const pair& start_index, const HandleGraph* graph, size_t start_rank, bool is_internal_node, size_t size_limit) { + /*Helper function to find the ancestor of a node that is a child of this snarl */ + auto get_ancestor_of_node = [&](pair curr_index, + pair ancestor_snarl_index) { - //Start from either direction for all nodes, but only going in for start and end - vector directions; - if (start_index.first == SnarlDistanceIndex::TEMP_NODE && start_index.second == temp_snarl_record.start_node_id) { - directions.emplace_back(temp_snarl_record.start_node_rev); - } else if (start_index.first == SnarlDistanceIndex::TEMP_NODE && start_index.second == temp_snarl_record.end_node_id){ - directions.emplace_back(!temp_snarl_record.end_node_rev); - } else { - directions.emplace_back(true); - directions.emplace_back(false); + //This is a child that isn't a node, so it must be a chain + if (curr_index.second == temp_snarl_record.start_node_id || + curr_index.second == temp_snarl_record.end_node_id) { + return curr_index; } - for (bool start_rev : directions) { - //Start a dijkstra traversal from start_index going in the direction indicated by start_rev - //Record the distances to each node (child of the snarl) found - size_t reachable_node_count = 0; //How many nodes can we reach from this node side? + //Otherwise, walk up until we hit the current snarl + pair parent_index = temp_index.temp_node_records.at(curr_index.second-temp_index.min_node_id).parent; + while (parent_index != ancestor_snarl_index) { + curr_index=parent_index; + parent_index = parent_index.first == SnarlDistanceIndex::TEMP_SNARL ? temp_index.temp_snarl_records.at(parent_index.second).parent + : temp_index.temp_chain_records.at(parent_index.second).parent; #ifdef debug_distance_indexing - cerr << " Starting from child " << temp_index.structure_start_end_as_string(start_index) - << " going " << (start_rev ? "rev" : "fd") << endl; + assert(parent_index.first != SnarlDistanceIndex::TEMP_ROOT); #endif + } + + return curr_index; + }; + + //Start from either direction for all nodes, but only going in for start and end + vector directions; + if (start_index.first == SnarlDistanceIndex::TEMP_NODE && start_index.second == temp_snarl_record.start_node_id) { + directions.emplace_back(temp_snarl_record.start_node_rev); + } else if (start_index.first == SnarlDistanceIndex::TEMP_NODE && start_index.second == temp_snarl_record.end_node_id){ + directions.emplace_back(!temp_snarl_record.end_node_rev); + } else { + directions.emplace_back(true); + directions.emplace_back(false); + } + for (bool start_rev : directions) { + //Start a dijkstra traversal from start_index going in the direction indicated by start_rev + //Record the distances to each node (child of the snarl) found + size_t reachable_node_count = 0; //How many nodes can we reach from this node side? - //Define a NetgraphNode as the value for the priority queue: - // , direction> - using NetgraphNode = pair, bool>>; - auto cmp = [] (const NetgraphNode a, const NetgraphNode b) { - return a.first > b.first; - }; - - //The priority queue of the next nodes to visit, ordered by the distance - std::priority_queue, decltype(cmp)> queue(cmp); - //The nodes we've already visited - unordered_set, bool>> visited_nodes; - visited_nodes.reserve(temp_snarl_record.node_count * 2); - - //Start from the current start node - queue.push(make_pair(0, make_pair(start_index, start_rev))); - - while (!queue.empty()) { - - //Get the current node from the queue and pop it out of the queue - size_t current_distance = queue.top().first; - pair current_index = queue.top().second.first; - bool current_rev = queue.top().second.second; - if (visited_nodes.count(queue.top().second)) { - queue.pop(); - continue; - } - visited_nodes.emplace(queue.top().second); +#ifdef debug_distance_indexing + cerr << " Starting from child " << temp_index.structure_start_end_as_string(start_index) + << " going " << (start_rev ? "rev" : "fd") << endl; +#endif + + //Define a NetgraphNode as the value for the priority queue: + // , direction> + using NetgraphNode = pair, bool>>; + auto cmp = [] (const NetgraphNode a, const NetgraphNode b) { + return a.first > b.first; + }; + + //The priority queue of the next nodes to visit, ordered by the distance + std::priority_queue, decltype(cmp)> queue(cmp); + //The nodes we've already visited + unordered_set, bool>> visited_nodes; + visited_nodes.reserve(temp_snarl_record.node_count * 2); + + //Start from the current start node + queue.push(make_pair(0, make_pair(start_index, start_rev))); + + while (!queue.empty()) { + + //Get the current node from the queue and pop it out of the queue + size_t current_distance = queue.top().first; + pair current_index = queue.top().second.first; + bool current_rev = queue.top().second.second; + if (visited_nodes.count(queue.top().second)) { queue.pop(); + continue; + } + visited_nodes.emplace(queue.top().second); + queue.pop(); - //The handle that we need to follow to get the next reachable nodes - //If the current node is a node, then its just the node. Otherwise, it's the - //opposite side of the child chain - handle_t current_end_handle = current_index.first == SnarlDistanceIndex::TEMP_NODE ? - graph->get_handle(current_index.second, current_rev) : - (current_rev ? graph->get_handle(temp_index.temp_chain_records[current_index.second].start_node_id, - !temp_index.temp_chain_records[current_index.second].start_node_rev) - : graph->get_handle(temp_index.temp_chain_records[current_index.second].end_node_id, - temp_index.temp_chain_records[current_index.second].end_node_rev)); + //The handle that we need to follow to get the next reachable nodes + //If the current node is a node, then its just the node. Otherwise, it's the + //opposite side of the child chain + handle_t current_end_handle = current_index.first == SnarlDistanceIndex::TEMP_NODE ? + graph->get_handle(current_index.second, current_rev) : + (current_rev ? graph->get_handle(temp_index.temp_chain_records[current_index.second].start_node_id, + !temp_index.temp_chain_records[current_index.second].start_node_rev) + : graph->get_handle(temp_index.temp_chain_records[current_index.second].end_node_id, + temp_index.temp_chain_records[current_index.second].end_node_rev)); #ifdef debug_distance_indexing - cerr << " at child " << temp_index.structure_start_end_as_string(current_index) << " going " - << (current_rev ? "rev" : "fd") << " at actual node " << graph->get_id(current_end_handle) - << (graph->get_is_reverse(current_end_handle) ? "rev" : "fd") << endl; + cerr << " at child " << temp_index.structure_start_end_as_string(current_index) << " going " + << (current_rev ? "rev" : "fd") << " at actual node " << graph->get_id(current_end_handle) + << (graph->get_is_reverse(current_end_handle) ? "rev" : "fd") << endl; #endif - graph->follow_edges(current_end_handle, false, [&](const handle_t next_handle) { - if (graph->get_id(current_end_handle) == graph->get_id(next_handle)){ - //If this loops onto the same node side then this isn't a simple snarl - temp_snarl_record.is_simple = false; - } else if ((current_index.first == SnarlDistanceIndex::TEMP_NODE ? current_index.second - : (current_rev ? temp_index.temp_chain_records[current_index.second].end_node_id - : temp_index.temp_chain_records[current_index.second].start_node_id)) - == graph->get_id(next_handle)){ - //If this loops to the other end of the chain then this isn't a simple snarl - temp_snarl_record.is_simple = false; - } else if (!temp_snarl_record.is_root_snarl && start_rank == 0 && - current_index != start_index && graph->get_id(next_handle) != temp_snarl_record.end_node_id) { - //If the starting point of this traversal was the start of the snarl, the current starting point is not the start node, - //and we found another child, then this is not a simple snarl - temp_snarl_record.is_simple = false; - } else if (!temp_snarl_record.is_root_snarl && start_rank == 1 && - current_index != start_index && graph->get_id(next_handle) != temp_snarl_record.start_node_id) { - //If the starting point of this traversal was the end of the snarl, the current starting point is not the end node, - //and we found another child, then this is not a simple snarl - temp_snarl_record.is_simple = false; - } + graph->follow_edges(current_end_handle, false, [&](const handle_t next_handle) { + if (graph->get_id(current_end_handle) == graph->get_id(next_handle)){ + //If this loops onto the same node side then this isn't a simple snarl + temp_snarl_record.is_simple = false; + } else if ((current_index.first == SnarlDistanceIndex::TEMP_NODE ? current_index.second + : (current_rev ? temp_index.temp_chain_records[current_index.second].end_node_id + : temp_index.temp_chain_records[current_index.second].start_node_id)) + == graph->get_id(next_handle)){ + //If this loops to the other end of the chain then this isn't a simple snarl + temp_snarl_record.is_simple = false; + } else if (!temp_snarl_record.is_root_snarl && start_rank == 0 && + current_index != start_index && graph->get_id(next_handle) != temp_snarl_record.end_node_id) { + //If the starting point of this traversal was the start of the snarl, the current starting point is not the start node, + //and we found another child, then this is not a simple snarl + temp_snarl_record.is_simple = false; + } else if (!temp_snarl_record.is_root_snarl && start_rank == 1 && + current_index != start_index && graph->get_id(next_handle) != temp_snarl_record.start_node_id) { + //If the starting point of this traversal was the end of the snarl, the current starting point is not the end node, + //and we found another child, then this is not a simple snarl + temp_snarl_record.is_simple = false; + } - reachable_node_count++; - //At each of the nodes reachable from the current one, fill in the distance from the start - //node to the next node (current_distance). If this handle isn't leaving the snarl, - //add the next nodes along with the distance to the end of the next node - auto& node_record = temp_index.temp_node_records.at(graph->get_id(next_handle)-temp_index.min_node_id); + reachable_node_count++; + //At each of the nodes reachable from the current one, fill in the distance from the start + //node to the next node (current_distance). If this handle isn't leaving the snarl, + //add the next nodes along with the distance to the end of the next node + auto& node_record = temp_index.temp_node_records.at(graph->get_id(next_handle)-temp_index.min_node_id); - //The index of the snarl's child that next_handle represents - pair next_index = - get_ancestor_of_node(make_pair(SnarlDistanceIndex::TEMP_NODE, graph->get_id(next_handle)), snarl_index); + //The index of the snarl's child that next_handle represents + pair next_index = + get_ancestor_of_node(make_pair(SnarlDistanceIndex::TEMP_NODE, graph->get_id(next_handle)), snarl_index); - bool next_is_tip = start_index.first == SnarlDistanceIndex::TEMP_NODE - ? temp_index.temp_node_records.at(start_index.second-temp_index.min_node_id).is_tip - : temp_index.temp_chain_records.at(start_index.second).is_tip; + bool next_is_tip = start_index.first == SnarlDistanceIndex::TEMP_NODE + ? temp_index.temp_node_records.at(start_index.second-temp_index.min_node_id).is_tip + : temp_index.temp_chain_records.at(start_index.second).is_tip; + + //The rank and orientation of next in the snarl + size_t next_rank = next_index.first == SnarlDistanceIndex::TEMP_NODE + ? node_record.rank_in_parent + : temp_index.temp_chain_records[next_index.second].rank_in_parent; + if (next_index.first == SnarlDistanceIndex::TEMP_NODE && next_index.second == temp_snarl_record.start_node_id) { + next_rank = 0; + } else if (next_index.first == SnarlDistanceIndex::TEMP_NODE && next_index.second == temp_snarl_record.end_node_id) { + next_rank = 1; + } else { + //If the next thing wasn't a boundary node and this was an internal node, then it isn't a simple snarl + if (is_internal_node) { + temp_snarl_record.is_simple = false; + } + }//TODO: This won't be true of root snarls + //else { + // assert(next_rank != 0 && next_rank != 1); + //} + bool next_rev = next_index.first == SnarlDistanceIndex::TEMP_NODE || temp_index.temp_chain_records[next_index.second].is_trivial + ? graph->get_is_reverse(next_handle) + : graph->get_id(next_handle) == temp_index.temp_chain_records[next_index.second].end_node_id; + + /**Record the distance **/ + bool start_is_boundary = !temp_snarl_record.is_root_snarl && (start_rank == 0 || start_rank == 1); + bool next_is_boundary = !temp_snarl_record.is_root_snarl && (next_rank == 0 || next_rank == 1); + + if (size_limit != 0 && + (temp_snarl_record.node_count < size_limit || start_is_boundary || next_is_boundary)) { + //If the snarl is too big, then we don't record distances between internal nodes + //If we are looking at all distances or we are looking at boundaries + bool added_new_distance = false; + + //Set the distance + pair start = start_is_boundary + ? make_pair(start_rank, false) : make_pair(start_rank, !start_rev); + pair next = next_is_boundary + ? make_pair(next_rank, false) : make_pair(next_rank, next_rev); + if (start_is_boundary && next_is_boundary) { + //If it is between bounds of the snarl, then the snarl stores it + if (start_rank == 0 && next_rank == 0 && + temp_snarl_record.distance_start_start == std::numeric_limits::max()) { + temp_snarl_record.distance_start_start = current_distance; + added_new_distance = true; + } else if (start_rank == 1 && next_rank == 1 && + temp_snarl_record.distance_end_end == std::numeric_limits::max()) { + temp_snarl_record.distance_end_end = current_distance; + added_new_distance = true; + } else if (((start_rank == 0 && next_rank == 1) || (start_rank == 1 && next_rank == 0)) + && temp_snarl_record.min_length == std::numeric_limits::max()){ + temp_snarl_record.min_length = current_distance; + added_new_distance = true; - //The rank and orientation of next in the snarl - size_t next_rank = next_index.first == SnarlDistanceIndex::TEMP_NODE - ? node_record.rank_in_parent - : temp_index.temp_chain_records[next_index.second].rank_in_parent; - if (next_index.first == SnarlDistanceIndex::TEMP_NODE && next_index.second == temp_snarl_record.start_node_id) { - next_rank = 0; - } else if (next_index.first == SnarlDistanceIndex::TEMP_NODE && next_index.second == temp_snarl_record.end_node_id) { - next_rank = 1; - } else { - //If the next thing wasn't a boundary node and this was an internal node, then it isn't a simple snarl - if (is_internal_node) { - temp_snarl_record.is_simple = false; } - }//TODO: This won't be true of root snarls - //else { - // assert(next_rank != 0 && next_rank != 1); - //} - bool next_rev = next_index.first == SnarlDistanceIndex::TEMP_NODE || temp_index.temp_chain_records[next_index.second].is_trivial - ? graph->get_is_reverse(next_handle) - : graph->get_id(next_handle) == temp_index.temp_chain_records[next_index.second].end_node_id; - - /**Record the distance **/ - bool start_is_boundary = !temp_snarl_record.is_root_snarl && (start_rank == 0 || start_rank == 1); - bool next_is_boundary = !temp_snarl_record.is_root_snarl && (next_rank == 0 || next_rank == 1); - - if (size_limit != 0 && - (temp_snarl_record.node_count < size_limit || start_is_boundary || next_is_boundary)) { - //If the snarl is too big, then we don't record distances between internal nodes - //If we are looking at all distances or we are looking at boundaries - bool added_new_distance = false; - - //Set the distance - pair start = start_is_boundary - ? make_pair(start_rank, false) : make_pair(start_rank, !start_rev); - pair next = next_is_boundary - ? make_pair(next_rank, false) : make_pair(next_rank, next_rev); - if (start_is_boundary && next_is_boundary) { - //If it is between bounds of the snarl, then the snarl stores it - if (start_rank == 0 && next_rank == 0 && - temp_snarl_record.distance_start_start == std::numeric_limits::max()) { - temp_snarl_record.distance_start_start = current_distance; - added_new_distance = true; - } else if (start_rank == 1 && next_rank == 1 && - temp_snarl_record.distance_end_end == std::numeric_limits::max()) { - temp_snarl_record.distance_end_end = current_distance; - added_new_distance = true; - } else if (((start_rank == 0 && next_rank == 1) || (start_rank == 1 && next_rank == 0)) - && temp_snarl_record.min_length == std::numeric_limits::max()){ - temp_snarl_record.min_length = current_distance; + } else if (start_is_boundary){ + //If start is a boundary node + if (next_index.first == SnarlDistanceIndex::TEMP_NODE) { + //Next is a node + auto& temp_node_record = temp_index.temp_node_records.at(next_index.second-temp_index.min_node_id); + if (start_rank == 0 && !next_rev && + temp_node_record.distance_left_start == std::numeric_limits::max()) { + temp_node_record.distance_left_start = current_distance; added_new_distance = true; - + } else if (start_rank == 0 && next_rev && + temp_node_record.distance_right_start == std::numeric_limits::max()) { + temp_node_record.distance_right_start = current_distance; + added_new_distance = true; + } else if (start_rank == 1 && !next_rev && + temp_node_record.distance_left_end == std::numeric_limits::max()) { + temp_node_record.distance_left_end = current_distance; + added_new_distance = true; + } else if (start_rank == 1 && next_rev && + temp_node_record.distance_right_end == std::numeric_limits::max()) { + temp_node_record.distance_right_end = current_distance; + added_new_distance = true; } - } else if (start_is_boundary){ - //If start is a boundary node - if (next_index.first == SnarlDistanceIndex::TEMP_NODE) { - //Next is a node - auto& temp_node_record = temp_index.temp_node_records.at(next_index.second-temp_index.min_node_id); - if (start_rank == 0 && !next_rev && - temp_node_record.distance_left_start == std::numeric_limits::max()) { - temp_node_record.distance_left_start = current_distance; - added_new_distance = true; - } else if (start_rank == 0 && next_rev && - temp_node_record.distance_right_start == std::numeric_limits::max()) { - temp_node_record.distance_right_start = current_distance; - added_new_distance = true; - } else if (start_rank == 1 && !next_rev && - temp_node_record.distance_left_end == std::numeric_limits::max()) { - temp_node_record.distance_left_end = current_distance; - added_new_distance = true; - } else if (start_rank == 1 && next_rev && - temp_node_record.distance_right_end == std::numeric_limits::max()) { - temp_node_record.distance_right_end = current_distance; - added_new_distance = true; - } - } else { - //Next is a chain - auto& temp_chain_record = temp_index.temp_chain_records.at(next_index.second); - if (start_rank == 0 && !next_rev && - temp_chain_record.distance_left_start == std::numeric_limits::max()) { - temp_chain_record.distance_left_start = current_distance; - added_new_distance = true; - } else if (start_rank == 0 && next_rev && - temp_chain_record.distance_right_start == std::numeric_limits::max()) { - temp_chain_record.distance_right_start = current_distance; - added_new_distance = true; - } else if (start_rank == 1 && !next_rev && - temp_chain_record.distance_left_end == std::numeric_limits::max()) { - temp_chain_record.distance_left_end = current_distance; - added_new_distance = true; - } else if (start_rank == 1 && next_rev && - temp_chain_record.distance_right_end == std::numeric_limits::max()) { - temp_chain_record.distance_right_end = current_distance; - added_new_distance = true; - } + } else { + //Next is a chain + auto& temp_chain_record = temp_index.temp_chain_records.at(next_index.second); + if (start_rank == 0 && !next_rev && + temp_chain_record.distance_left_start == std::numeric_limits::max()) { + temp_chain_record.distance_left_start = current_distance; + added_new_distance = true; + } else if (start_rank == 0 && next_rev && + temp_chain_record.distance_right_start == std::numeric_limits::max()) { + temp_chain_record.distance_right_start = current_distance; + added_new_distance = true; + } else if (start_rank == 1 && !next_rev && + temp_chain_record.distance_left_end == std::numeric_limits::max()) { + temp_chain_record.distance_left_end = current_distance; + added_new_distance = true; + } else if (start_rank == 1 && next_rev && + temp_chain_record.distance_right_end == std::numeric_limits::max()) { + temp_chain_record.distance_right_end = current_distance; + added_new_distance = true; } - } else if (!next_is_boundary && !temp_snarl_record.distances.count(make_pair(start, next))) { - //Otherwise the snarl stores it in its distance - //If the distance isn't from an internal node to a bound and we haven't stored the distance yet + } + } else if (!next_is_boundary && !temp_snarl_record.distances.count(make_pair(start, next))) { + //Otherwise the snarl stores it in its distance + //If the distance isn't from an internal node to a bound and we haven't stored the distance yet - temp_snarl_record.distances[make_pair(start, next)] = current_distance; - added_new_distance = true; + temp_snarl_record.distances[make_pair(start, next)] = current_distance; + added_new_distance = true; #ifdef debug_distance_indexing - cerr << " Adding distance between ranks " << start.first << " " << start.second << " and " << next.first << " " << next.second << ": " << current_distance << endl; + cerr << " Adding distance between ranks " << start.first << " " << start.second << " and " << next.first << " " << next.second << ": " << current_distance << endl; #endif - } - if (added_new_distance) { - temp_snarl_record.max_distance = std::max(temp_snarl_record.max_distance, current_distance); - } } + if (added_new_distance) { + temp_snarl_record.max_distance = std::max(temp_snarl_record.max_distance, current_distance); + } + } - /**Add the next node to the priority queue**/ - - if (visited_nodes.count(make_pair(next_index, next_rev)) == 0 && + /**Add the next node to the priority queue**/ + + if (visited_nodes.count(make_pair(next_index, next_rev)) == 0 && + graph->get_id(next_handle) != temp_snarl_record.start_node_id && + graph->get_id(next_handle) != temp_snarl_record.end_node_id + ) { + //If this isn't leaving the snarl, + //then add the next node to the queue, along with the distance to traverse it + size_t next_node_length = next_index.first == SnarlDistanceIndex::TEMP_NODE ? graph->get_length(next_handle) : + temp_index.temp_chain_records[next_index.second].min_length; + if (next_index.first == SnarlDistanceIndex::TEMP_CHAIN && + temp_index.temp_chain_records[next_index.second].chain_components.back() != 0) { + //If there are multiple components, then the chain is not start-end reachable so its length + //is actually infinite + next_node_length = std::numeric_limits::max(); + } + if (next_node_length != std::numeric_limits::max()) { + queue.push(make_pair(SnarlDistanceIndex::sum(current_distance, next_node_length), + make_pair(next_index, next_rev))); + } + } + if (next_index.first == SnarlDistanceIndex::TEMP_CHAIN) { + size_t loop_distance = next_rev ? temp_index.temp_chain_records[next_index.second].backward_loops.back() + : temp_index.temp_chain_records[next_index.second].forward_loops.front(); + if (loop_distance != std::numeric_limits::max() && + visited_nodes.count(make_pair(next_index, !next_rev)) == 0 && graph->get_id(next_handle) != temp_snarl_record.start_node_id && graph->get_id(next_handle) != temp_snarl_record.end_node_id ) { - //If this isn't leaving the snarl, - //then add the next node to the queue, along with the distance to traverse it - size_t next_node_length = next_index.first == SnarlDistanceIndex::TEMP_NODE ? graph->get_length(next_handle) : - temp_index.temp_chain_records[next_index.second].min_length; - if (next_index.first == SnarlDistanceIndex::TEMP_CHAIN && - temp_index.temp_chain_records[next_index.second].chain_components.back() != 0) { - //If there are multiple components, then the chain is not start-end reachable so its length - //is actually infinite - next_node_length = std::numeric_limits::max(); - } - if (next_node_length != std::numeric_limits::max()) { - queue.push(make_pair(SnarlDistanceIndex::sum(current_distance, next_node_length), - make_pair(next_index, next_rev))); - } - } - if (next_index.first == SnarlDistanceIndex::TEMP_CHAIN) { - size_t loop_distance = next_rev ? temp_index.temp_chain_records[next_index.second].backward_loops.back() - : temp_index.temp_chain_records[next_index.second].forward_loops.front(); - if (loop_distance != std::numeric_limits::max() && - visited_nodes.count(make_pair(next_index, !next_rev)) == 0 && - graph->get_id(next_handle) != temp_snarl_record.start_node_id && - graph->get_id(next_handle) != temp_snarl_record.end_node_id - ) { - //If the next node can loop back on itself, then add the next node in the opposite direction - size_t next_node_len = loop_distance + 2 * graph->get_length(next_handle); - queue.push(make_pair(SnarlDistanceIndex::sum(current_distance, next_node_len), - make_pair(next_index, !next_rev))); - } + //If the next node can loop back on itself, then add the next node in the opposite direction + size_t next_node_len = loop_distance + 2 * graph->get_length(next_handle); + queue.push(make_pair(SnarlDistanceIndex::sum(current_distance, next_node_len), + make_pair(next_index, !next_rev))); } + } #ifdef debug_distance_indexing - cerr << " reached child " << temp_index.structure_start_end_as_string(next_index) << "going " - << (next_rev ? "rev" : "fd") << " with distance " << current_distance << " for ranks " << start_rank << " " << next_rank << endl; + cerr << " reached child " << temp_index.structure_start_end_as_string(next_index) << "going " + << (next_rev ? "rev" : "fd") << " with distance " << current_distance << " for ranks " << start_rank << " " << next_rank << endl; #endif - }); - } - if (is_internal_node && reachable_node_count != 1) { - //If this is an internal node, then it must have only one edge for it to be a simple snarl - temp_snarl_record.is_simple = false; - } + }); } - - /** Check the minimum length of the snarl passing through this node **/ - if (start_rank != 0 && start_rank != 1) { - - size_t child_max_length = start_index.first == SnarlDistanceIndex::TEMP_NODE - ? temp_index.temp_node_records.at(start_index.second-temp_index.min_node_id).node_length - : temp_index.temp_chain_records.at(start_index.second).max_length; - //The distance through the whole snarl traversing this node forwards - //(This might actually be traversing it backwards but it doesn't really matter) - - size_t dist_start_left = start_index.first == SnarlDistanceIndex::TEMP_NODE - ? temp_index.temp_node_records.at(start_index.second-temp_index.min_node_id).distance_left_start - : temp_index.temp_chain_records.at(start_index.second).distance_left_start; - size_t dist_end_right = start_index.first == SnarlDistanceIndex::TEMP_NODE - ? temp_index.temp_node_records.at(start_index.second-temp_index.min_node_id).distance_right_end - : temp_index.temp_chain_records.at(start_index.second).distance_right_end; - size_t dist_start_right = start_index.first == SnarlDistanceIndex::TEMP_NODE - ? temp_index.temp_node_records.at(start_index.second-temp_index.min_node_id).distance_right_start - : temp_index.temp_chain_records.at(start_index.second).distance_right_start; - size_t dist_end_left = start_index.first == SnarlDistanceIndex::TEMP_NODE - ? temp_index.temp_node_records.at(start_index.second-temp_index.min_node_id).distance_left_end - : temp_index.temp_chain_records.at(start_index.second).distance_left_end; - - size_t snarl_length_fd = SnarlDistanceIndex::sum(SnarlDistanceIndex::sum( - dist_start_left, dist_end_right),child_max_length); - //The same thing traversing this node backwards - size_t snarl_length_rev = SnarlDistanceIndex::sum(SnarlDistanceIndex::sum( - dist_start_right, dist_end_left), child_max_length); - //The max that isn't infinite - size_t max_length = - snarl_length_rev == std::numeric_limits::max() - ? snarl_length_fd - : (snarl_length_fd == std::numeric_limits::max() - ? snarl_length_rev - : std::max(snarl_length_rev, snarl_length_fd)); - if (max_length != std::numeric_limits::max()) { - temp_snarl_record.max_length = std::max(temp_snarl_record.max_length, max_length); - } - if ( temp_snarl_record.is_simple && - ! ((dist_start_left == 0 && dist_end_right == 0 && dist_end_left == std::numeric_limits::max() && dist_start_right == std::numeric_limits::max() ) || - (dist_start_left == std::numeric_limits::max() && dist_end_right == std::numeric_limits::max() && dist_end_left == 0 && dist_start_right == 0 ))){ - //If the snarl is simple, double check that this node is actually simple: that it can only be traversed going - //across the nsarl - temp_snarl_record.is_simple = false; - } + if (is_internal_node && reachable_node_count != 1) { + //If this is an internal node, then it must have only one edge for it to be a simple snarl + temp_snarl_record.is_simple = false; } } - - //If this is a simple snarl (one with only single nodes that connect to the start and end nodes), then - // we want to remember if the child nodes are reversed - if (temp_snarl_record.is_simple) { - for (size_t i = 0 ; i < temp_snarl_record.node_count ; i++) { - //Get the index of the child - const pair& child_index = temp_snarl_record.children[i]; - //Which is a node -#ifdef debug_distance_indexing - assert(child_index.first == SnarlDistanceIndex::TEMP_NODE); -#endif - - //And get the record - SnarlDistanceIndex::TemporaryDistanceIndex::TemporaryNodeRecord& temp_node_record = - temp_index.temp_node_records[child_index.second-temp_index.min_node_id]; - size_t rank =temp_node_record.rank_in_parent; - - - - //Set the orientation of this node in the simple snarl - temp_node_record.reversed_in_parent = temp_node_record.distance_left_start == std::numeric_limits::max(); - + /** Check the minimum length of the snarl passing through this node **/ + if (start_rank != 0 && start_rank != 1) { + + size_t child_max_length = start_index.first == SnarlDistanceIndex::TEMP_NODE + ? temp_index.temp_node_records.at(start_index.second-temp_index.min_node_id).node_length + : temp_index.temp_chain_records.at(start_index.second).max_length; + //The distance through the whole snarl traversing this node forwards + //(This might actually be traversing it backwards but it doesn't really matter) + + size_t dist_start_left = start_index.first == SnarlDistanceIndex::TEMP_NODE + ? temp_index.temp_node_records.at(start_index.second-temp_index.min_node_id).distance_left_start + : temp_index.temp_chain_records.at(start_index.second).distance_left_start; + size_t dist_end_right = start_index.first == SnarlDistanceIndex::TEMP_NODE + ? temp_index.temp_node_records.at(start_index.second-temp_index.min_node_id).distance_right_end + : temp_index.temp_chain_records.at(start_index.second).distance_right_end; + size_t dist_start_right = start_index.first == SnarlDistanceIndex::TEMP_NODE + ? temp_index.temp_node_records.at(start_index.second-temp_index.min_node_id).distance_right_start + : temp_index.temp_chain_records.at(start_index.second).distance_right_start; + size_t dist_end_left = start_index.first == SnarlDistanceIndex::TEMP_NODE + ? temp_index.temp_node_records.at(start_index.second-temp_index.min_node_id).distance_left_end + : temp_index.temp_chain_records.at(start_index.second).distance_left_end; + + size_t snarl_length_fd = SnarlDistanceIndex::sum(SnarlDistanceIndex::sum( + dist_start_left, dist_end_right),child_max_length); + //The same thing traversing this node backwards + size_t snarl_length_rev = SnarlDistanceIndex::sum(SnarlDistanceIndex::sum( + dist_start_right, dist_end_left), child_max_length); + //The max that isn't infinite + size_t max_length = + snarl_length_rev == std::numeric_limits::max() + ? snarl_length_fd + : (snarl_length_fd == std::numeric_limits::max() + ? snarl_length_rev + : std::max(snarl_length_rev, snarl_length_fd)); + if (max_length != std::numeric_limits::max()) { + temp_snarl_record.max_length = std::max(temp_snarl_record.max_length, max_length); + } + if ( temp_snarl_record.is_simple && + ! ((dist_start_left == 0 && dist_end_right == 0 && dist_end_left == std::numeric_limits::max() && dist_start_right == std::numeric_limits::max() ) || + (dist_start_left == std::numeric_limits::max() && dist_end_right == std::numeric_limits::max() && dist_end_left == 0 && dist_start_right == 0 ))){ + //If the snarl is simple, double check that this node is actually simple: that it can only be traversed going + //across the nsarl + temp_snarl_record.is_simple = false; } } - - //Now that the distances are filled in, predict the size of the snarl in the index - temp_index.max_index_size += temp_snarl_record.get_max_record_length(); - if (temp_snarl_record.is_simple) { - temp_index.max_index_size -= (temp_snarl_record.children.size() * SnarlDistanceIndex::TemporaryDistanceIndex::TemporaryNodeRecord::get_max_record_length()); - } - - // For simple snarl records, need 11 + 11 + number of bits for the number of children - temp_index.max_bits = std::max(temp_index.max_bits, 22 + SnarlDistanceIndex::bit_width(temp_snarl_record.children.size())); } + //Given an alignment to a graph and a range, find the set of nodes in the //graph for which the minimum distance from the position to any position //in the node is within the given distance range From 297a11f90f5b25f1f6e0079c9458f507ca0884e9 Mon Sep 17 00:00:00 2001 From: Zia Truong <194475824+electricEpilith@users.noreply.github.com> Date: Fri, 23 Jan 2026 08:36:01 -0800 Subject: [PATCH 02/77] hub labeling in (debugging not finished), also changes to deal with C++20 upgrade --- Makefile | 4 ++-- src/snarl_distance_index.cpp | 30 +++++++++++++++------------ src/snarl_distance_index.hpp | 1 + src/subcommand/haplotypes_main.cpp | 20 +++++++++--------- src/subcommand/inject_main.cpp | 2 +- src/subcommand/minimizer_main.cpp | 2 +- src/unittest/packed_structs.cpp | 8 +++---- src/unittest/snarl_distance_index.cpp | 19 +++++++++++++++++ 8 files changed, 55 insertions(+), 31 deletions(-) diff --git a/Makefile b/Makefile index a5af18915e..ac203621f1 100644 --- a/Makefile +++ b/Makefile @@ -227,8 +227,8 @@ else $(info OS is Linux) $(info Compiler $(CXX) is assumed to be GCC) - # Linux can have some old compilers so we want to work back to C++14 - CXX_STANDARD?=14 + # C++20 for spaceship operator and ranges + CXX_STANDARD?=20 # Set an rpath for vg and dependency utils to find installed libraries LD_UTIL_RPATH_FLAGS="-Wl,-rpath,$(CWD)/$(LIB_DIR)" diff --git a/src/snarl_distance_index.cpp b/src/snarl_distance_index.cpp index c6384a9f2b..1df914b5ef 100644 --- a/src/snarl_distance_index.cpp +++ b/src/snarl_distance_index.cpp @@ -1,4 +1,4 @@ -//#define debug_distance_indexing +#define debug_distance_indexing //#define debug_snarl_traversal //#define debug_distances //#define debug_subgraph @@ -790,7 +790,7 @@ Fills in required distance matrix rows for each child - size_limit == 0: no distances in index, so no rows - Top-level chain distances only: ??? */ -static void populate_distance_matrix_if_needed(SnarlDistanceIndex::TemporaryDistanceIndex& temp_index, pair& snarl_index, SnarlDistanceIndex::TemporaryDistanceIndex::TemporarySnarlRecord& temp_snarl_record, vector>& all_children, const HandleGraph* graph, size_t size_limit); +static void populate_distance_matrix_if_needed(SnarlDistanceIndex::TemporaryDistanceIndex& temp_index, pair& snarl_index, SnarlDistanceIndex::TemporaryDistanceIndex::TemporarySnarlRecord& temp_snarl_record, vector>& all_children, const HandleGraph* graph, size_t size_limit, bool only_top_level_chain_distances); /* Does three things: @@ -1115,9 +1115,6 @@ void populate_snarl_index( * Start a dijkstra traversal from each node side in the snarl and record all distances */ - if (size_limit != 0 && temp_snarl_record.node_count > size_limit) { - temp_index.use_oversized_snarls = true; - } //Add the start and end nodes to the list of children so that we include them in the traversal if (!temp_snarl_record.is_root_snarl) { @@ -1125,15 +1122,21 @@ void populate_snarl_index( all_children.emplace_back(SnarlDistanceIndex::TEMP_NODE, temp_snarl_record.end_node_id); } - if (size_limit == 0) { - temp_snarl_record.include_distances = false; - } else if (only_top_level_chain_distances) { - temp_snarl_record.include_distances = false; - } else if (temp_index.use_oversized_snarls) { + #ifdef debug_distance_indexing + cerr << "is_simple: " << temp_snarl_record.is_simple << endl; + #endif + + if (size_limit != 0 && temp_snarl_record.node_count > size_limit) { + temp_index.use_oversized_snarls = true; + temp_snarl_record.is_simple = false; populate_hub_labeling(temp_index, snarl_index, temp_snarl_record, all_children, graph); } else { - populate_distance_matrix_if_needed(temp_index, snarl_index, temp_snarl_record, all_children, graph, size_limit); - } + if (size_limit == 0 || only_top_level_chain_distances) { + temp_snarl_record.include_distances = false; + } + //also sets is_simple to false if snarl isn't simple + populate_distance_matrix_if_needed(temp_index, snarl_index, temp_snarl_record, all_children, graph, size_limit, only_top_level_chain_distances); + } //If this is a simple snarl (one with only single nodes that connect to the start and end nodes), then // we want to remember if the child nodes are reversed @@ -1177,9 +1180,10 @@ void populate_hub_labeling(SnarlDistanceIndex::TemporaryDistanceIndex& temp_inde vector> labels_rev; labels_rev.resize(num_vertices(ov)); create_labels(labels, labels_rev, ov); //TODO: Put labels in temp_snarl_record + temp_snarl_record.hub_labels = pack_labels(labels, labels_rev); } -void populate_distance_matrix_if_needed(SnarlDistanceIndex::TemporaryDistanceIndex& temp_index, pair& snarl_index, SnarlDistanceIndex::TemporaryDistanceIndex::TemporarySnarlRecord& temp_snarl_record, vector>& all_children, const HandleGraph* graph, size_t size_limit) { +void populate_distance_matrix_if_needed(SnarlDistanceIndex::TemporaryDistanceIndex& temp_index, pair& snarl_index, SnarlDistanceIndex::TemporaryDistanceIndex::TemporarySnarlRecord& temp_snarl_record, vector>& all_children, const HandleGraph* graph, size_t size_limit, bool only_top_level_chain_distances) { if (size_limit != 0 && !only_top_level_chain_distances) { //If we are saving distances //Reserve enough space to store all possible distances diff --git a/src/snarl_distance_index.hpp b/src/snarl_distance_index.hpp index 43268d4b23..af38cea819 100644 --- a/src/snarl_distance_index.hpp +++ b/src/snarl_distance_index.hpp @@ -2,6 +2,7 @@ #define VG_SNARL_DISTANCE_HPP_INCLUDED #include +#include #include "snarls.hpp" #include #include "hash_map.hpp" diff --git a/src/subcommand/haplotypes_main.cpp b/src/subcommand/haplotypes_main.cpp index 6723666d4c..af3e938acd 100644 --- a/src/subcommand/haplotypes_main.cpp +++ b/src/subcommand/haplotypes_main.cpp @@ -889,7 +889,7 @@ void validate_error_sequence(size_t chain_id, size_t subchain_id, size_t sequenc } std::string validate_unary_path(const HandleGraph& graph, handle_t from, handle_t to) { - hash_set visited; + vg::hash_set visited; handle_t curr = from; while (curr != to) { if (visited.find(curr) != visited.end()) { @@ -913,7 +913,7 @@ std::string validate_unary_path(const HandleGraph& graph, handle_t from, handle_ // Returns true if the path from (start, offset) reaches the end without revisiting start or leaving the subchain. // The path may continue in subsequent fragments. bool trace_path( - const gbwt::GBWT& index, const gbwt::FragmentMap& fragment_map, const hash_set& subchain_nodes, + const gbwt::GBWT& index, const gbwt::FragmentMap& fragment_map, const vg::hash_set& subchain_nodes, gbwt::size_type sequence_id, gbwt::node_type start, gbwt::size_type offset, gbwt::node_type end ) { gbwt::edge_type pos(start, offset); @@ -1055,8 +1055,8 @@ void validate_chain(const Haplotypes::TopLevelChain& chain, // Sequences: normal subchains. if (subchain.type == Haplotypes::Subchain::normal) { std::vector da = r_index.decompressDA(subchain.start); - hash_set nodes = extract_subchain(graph, gbwtgraph::GBWTGraph::node_to_handle(subchain.start), gbwtgraph::GBWTGraph::node_to_handle(subchain.end)); - hash_set selected; + vg::hash_set nodes = extract_subchain(graph, gbwtgraph::GBWTGraph::node_to_handle(subchain.start), gbwtgraph::GBWTGraph::node_to_handle(subchain.end)); + vg::hash_set selected; for (size_t i = 0; i < da.size(); i++) { if (trace_path(*(graph.index), fragment_map, nodes, da[i], subchain.start, i, subchain.end)) { selected.insert(Haplotypes::sequence_type(da[i], i)); @@ -1082,7 +1082,7 @@ void validate_chain(const Haplotypes::TopLevelChain& chain, std::string message = expected_got(da.size(), subchain.sequences.size()) + " sequences (prefix / suffix)"; validate_error_subchain(chain_id, subchain_id, message); } - hash_set truth; + vg::hash_set truth; for (size_t i = 0; i < da.size(); i++) { truth.insert({ da[i], i }); } @@ -1103,7 +1103,7 @@ void validate_chain(const Haplotypes::TopLevelChain& chain, // Kmers. if (subchain.type != Haplotypes::Subchain::full_haplotype) { - hash_set all_kmers; + vg::hash_set all_kmers; for (size_t i = 0; i < subchain.kmers.size(); i++) { all_kmers.insert(subchain.kmers[i]); } @@ -1111,14 +1111,14 @@ void validate_chain(const Haplotypes::TopLevelChain& chain, std::string message = expected_got(subchain.kmers.size(), all_kmers.size()) + " kmers"; validate_error_subchain(chain_id, subchain_id, message); } - hash_map used_kmers; // (kmer used in haplotypes, number of sequences that contain it) - hash_map missing_kmers; // (kmer not used in haplotypes, number of sequences that contain it) + vg::hash_map used_kmers; // (kmer used in haplotypes, number of sequences that contain it) + vg::hash_map missing_kmers; // (kmer not used in haplotypes, number of sequences that contain it) for (size_t i = 0; i < subchain.sequences.size(); i++) { std::vector haplotype = get_haplotype( graph, fragment_map, subchain.sequences[i], subchain.start, subchain.end, minimizer_index.k() ); - hash_map unique_minimizers; // (kmer, used in the sequence) + vg::hash_map unique_minimizers; // (kmer, used in the sequence) for (const std::string& sequence : haplotype) { auto minimizers = minimizer_index.minimizers(sequence); for (auto& minimizer : minimizers) { @@ -1238,7 +1238,7 @@ void validate_haplotypes(const Haplotypes& haplotypes, if (verbosity >= HaplotypePartitioner::Verbosity::verbosity_detailed) { std::cerr << "Validating kmer specificity" << std::endl; } - hash_map> kmers; + vg::hash_map> kmers; size_t collisions = 0, total_kmers = 0; for (size_t chain_id = 0; chain_id < haplotypes.components(); chain_id++) { const Haplotypes::TopLevelChain& chain = haplotypes.chains[chain_id]; diff --git a/src/subcommand/inject_main.cpp b/src/subcommand/inject_main.cpp index 61f0a87738..4921b267eb 100644 --- a/src/subcommand/inject_main.cpp +++ b/src/subcommand/inject_main.cpp @@ -143,7 +143,7 @@ int main_inject(int argc, char** argv) { set_crash_context(aln.name()); if (add_identity) { // Calculate & save identity statistic - aln.set_identity(identity(aln.path())); + aln.set_identity(vg::identity(aln.path())); } if (rescore) { // Rescore the alignment diff --git a/src/subcommand/minimizer_main.cpp b/src/subcommand/minimizer_main.cpp index 0ca765007e..5dc331bfc8 100644 --- a/src/subcommand/minimizer_main.cpp +++ b/src/subcommand/minimizer_main.cpp @@ -174,7 +174,7 @@ void construct_minimizer_dispatch( ZipCodeCollection oversized_zipcodes; //Map node id to what gets stored in the payload - either the zipcode or index into oversized_zipcodes - hash_map node_id_to_payload; + vg::hash_map node_id_to_payload; node_id_to_payload.reserve(gbz->graph.max_node_id() - gbz->graph.min_node_id()); // Build the index. diff --git a/src/unittest/packed_structs.cpp b/src/unittest/packed_structs.cpp index 22f4f77a9b..588a45176a 100644 --- a/src/unittest/packed_structs.cpp +++ b/src/unittest/packed_structs.cpp @@ -69,7 +69,7 @@ using namespace std; case APPEND: for (size_t k = 0; k < appends_per_op; k++) { std_vec.push_back(next_val); - dyn_vec.append(next_val); + dyn_vec.push_back(next_val); next_val++; } @@ -79,7 +79,7 @@ using namespace std; if (!std_vec.empty()) { for (size_t k = 0; k < pops_per_op; k++) { std_vec.pop_back(); - dyn_vec.pop(); + dyn_vec.pop_back(); } } @@ -161,7 +161,7 @@ using namespace std; case APPEND: for (size_t k = 0; k < appends_per_op; k++) { std_vec.push_back(next_val); - dyn_vec.append(next_val); + dyn_vec.append_back(next_val); next_val = val_distr(prng); } @@ -171,7 +171,7 @@ using namespace std; if (!std_vec.empty()) { for (size_t k = 0; k < pops_per_op; k++) { std_vec.pop_back(); - dyn_vec.pop(); + dyn_vec.pop_back(); } } diff --git a/src/unittest/snarl_distance_index.cpp b/src/unittest/snarl_distance_index.cpp index 8e184ce4bd..f28c8c1416 100644 --- a/src/unittest/snarl_distance_index.cpp +++ b/src/unittest/snarl_distance_index.cpp @@ -6493,6 +6493,25 @@ namespace vg { } } + TEST_CASE( "Tiny oversized snarl", "[snarl_distance]" ) { + VG graph; + handle_t n1 = graph.create_handle("GCA"); + handle_t n2 = graph.create_handle("T"); + handle_t n3 = graph.create_handle("G"); + handle_t n4 = graph.create_handle("CTGA"); + + graph.create_edge(n1, n2); + graph.create_edge(n1, n3); + graph.create_edge(n2, n3); + graph.create_edge(n2, n4); + graph.create_edge(n3, n4); + IntegratedSnarlFinder snarl_finder(graph); + SnarlDistanceIndex distance_index; + fill_in_distance_index(&distance_index, &graph, &snarl_finder, 1); + + REQUIRE(distance_index.minimum_distance(2, false, 0, 3, false, 0, false, &graph) == 1); + } + TEST_CASE( "Oversized snarl","[snarl_distance]" ) { VG graph; From 9d4c2e251c16b85412d22df35fae02f900fed9f1 Mon Sep 17 00:00:00 2001 From: Adam Novak Date: Fri, 23 Jan 2026 16:36:16 -0500 Subject: [PATCH 03/77] Point at compatible libbdsg and get build working on Mac --- Makefile | 3 +- deps/libbdsg | 2 +- src/cluster.hpp | 82 ++++++++++++++++++++++++------------------------- 3 files changed, 44 insertions(+), 43 deletions(-) diff --git a/Makefile b/Makefile index ac203621f1..e6f21222d0 100644 --- a/Makefile +++ b/Makefile @@ -103,7 +103,8 @@ ifeq ($(shell uname -s),Darwin) LD_UTIL_RPATH_FLAGS="" # Homebrew installs a Protobuf that uses an Abseil that is built with C++17, so we need to build with at least C++17 - CXX_STANDARD?=17 + # C++20 for spaceship operator and ranges + CXX_STANDARD?=20 # We may need libraries from Macports ifeq ($(shell if [ -d /opt/local/lib ];then echo 1;else echo 0;fi), 1) diff --git a/deps/libbdsg b/deps/libbdsg index 47a77f821d..37492aa4f0 160000 --- a/deps/libbdsg +++ b/deps/libbdsg @@ -1 +1 @@ -Subproject commit 47a77f821d15603b434b2610de46f7c24c6b9af3 +Subproject commit 37492aa4f032f51f7a2ed2d27d7d8212eac228de diff --git a/src/cluster.hpp b/src/cluster.hpp index df997cc51c..cd6deab517 100644 --- a/src/cluster.hpp +++ b/src/cluster.hpp @@ -212,8 +212,8 @@ class MEMClusterer { protected: - class HitNode; class HitEdge; + class HitNode; class HitGraph; class DPScoreComparator; @@ -232,7 +232,47 @@ class MEMClusterer { /// is closest to the optimal separation void deduplicate_cluster_pairs(vector, int64_t>>& cluster_pairs, int64_t optimal_separation); }; + +class MEMClusterer::HitEdge { +public: + HitEdge(size_t to_idx, int32_t weight, int64_t distance) : to_idx(to_idx), weight(weight), distance(distance) {} + HitEdge() = default; + ~HitEdge() = default; + + /// Index of the node that the edge points to + size_t to_idx; + /// Weight for dynamic programming + int32_t weight; + + /// Estimated distance + int64_t distance; +}; + +class MEMClusterer::HitNode { +public: + HitNode(const MaximalExactMatch& mem, pos_t start_pos, int32_t score) : mem(&mem), start_pos(start_pos), score(score) { } + HitNode() = default; + ~HitNode() = default; + + const MaximalExactMatch* mem; + + /// Position of GCSA hit in the graph + pos_t start_pos; + + /// Score of the exact match this node represents + int32_t score; + + /// Score used in dynamic programming + int32_t dp_score; + + /// Edges from this node that are colinear with the read + vector edges_from; + + /// Edges to this node that are colinear with the read + vector edges_to; +}; + class MEMClusterer::HitGraph { public: @@ -286,46 +326,6 @@ class MEMClusterer::HitGraph { UnionFind components; }; -class MEMClusterer::HitNode { -public: - HitNode(const MaximalExactMatch& mem, pos_t start_pos, int32_t score) : mem(&mem), start_pos(start_pos), score(score) { } - HitNode() = default; - ~HitNode() = default; - - const MaximalExactMatch* mem; - - /// Position of GCSA hit in the graph - pos_t start_pos; - - /// Score of the exact match this node represents - int32_t score; - - /// Score used in dynamic programming - int32_t dp_score; - - /// Edges from this node that are colinear with the read - vector edges_from; - - /// Edges to this node that are colinear with the read - vector edges_to; -}; - -class MEMClusterer::HitEdge { -public: - HitEdge(size_t to_idx, int32_t weight, int64_t distance) : to_idx(to_idx), weight(weight), distance(distance) {} - HitEdge() = default; - ~HitEdge() = default; - - /// Index of the node that the edge points to - size_t to_idx; - - /// Weight for dynamic programming - int32_t weight; - - /// Estimated distance - int64_t distance; -}; - struct MEMClusterer::DPScoreComparator { private: const vector& nodes; From 8c13cf3357a32c5f28beb58458aed29f95f77a9c Mon Sep 17 00:00:00 2001 From: Adam Novak Date: Fri, 23 Jan 2026 18:10:56 -0500 Subject: [PATCH 04/77] Use the new indexing types and accessors to avoid fetching nodes by the wrong thing --- deps/libbdsg | 2 +- src/snarl_distance_index.cpp | 288 ++++++++++++++++++----------------- src/snarl_distance_index.hpp | 2 +- 3 files changed, 151 insertions(+), 141 deletions(-) diff --git a/deps/libbdsg b/deps/libbdsg index 37492aa4f0..0065e9717c 160000 --- a/deps/libbdsg +++ b/deps/libbdsg @@ -1 +1 @@ -Subproject commit 37492aa4f032f51f7a2ed2d27d7d8212eac228de +Subproject commit 0065e9717c949c878427fe39aacb977fc2508e2d diff --git a/src/snarl_distance_index.cpp b/src/snarl_distance_index.cpp index 1df914b5ef..9d65aefc90 100644 --- a/src/snarl_distance_index.cpp +++ b/src/snarl_distance_index.cpp @@ -91,7 +91,7 @@ SnarlDistanceIndex::TemporaryDistanceIndex make_temporary_distance_index( //Stores unfinished records, as type of record and offset into appropriate vector //(temp_node/snarl/chain_records) - vector> stack; + vector stack; //There may be components of the root that are connected to each other. Each connected component will //get put into a (fake) root-level snarl, but we don't know what those components will be initially, @@ -112,7 +112,7 @@ SnarlDistanceIndex::TemporaryDistanceIndex make_temporary_distance_index( #ifdef debug_distance_indexing cerr << " Starting new chain at " << graph->get_id(chain_start_handle) << (graph->get_is_reverse(chain_start_handle) ? " reverse" : " forward") << endl; //We shouldn't have seen this node before - //assert(temp_index.temp_node_records[graph->get_id(chain_start_handle)-min_node_id].node_id == 0); + //assert(temp_index.get_node(make_pair(SnarlDistanceIndex::TEMP_NODE, graph->get_id(chain_start_handle))).node_id == 0); #endif //Fill in node in chain @@ -126,7 +126,7 @@ SnarlDistanceIndex::TemporaryDistanceIndex make_temporary_distance_index( //And the node record itself - auto& temp_node = temp_index.temp_node_records.at(node_id-temp_index.min_node_id); + auto& temp_node = temp_index.get_node(temp_chain.children.back()); temp_node.node_id = node_id; temp_node.node_length = graph->get_length(chain_start_handle); temp_node.reversed_in_parent = graph->get_is_reverse(chain_start_handle); @@ -140,13 +140,13 @@ SnarlDistanceIndex::TemporaryDistanceIndex make_temporary_distance_index( */ //Done with this chain - pair chain_index = stack.back(); + SnarlDistanceIndex::temp_record_ref_t chain_index = stack.back(); stack.pop_back(); #ifdef debug_distance_indexing assert(chain_index.first == SnarlDistanceIndex::TEMP_CHAIN); #endif - SnarlDistanceIndex::TemporaryDistanceIndex::TemporaryChainRecord& temp_chain_record = temp_index.temp_chain_records.at(chain_index.second); + SnarlDistanceIndex::TemporaryDistanceIndex::TemporaryChainRecord& temp_chain_record = temp_index.get_chain(chain_index); nid_t node_id = graph->get_id(chain_end_handle); if (temp_chain_record.children.size() == 1 && node_id == temp_chain_record.start_node_id) { @@ -158,7 +158,8 @@ SnarlDistanceIndex::TemporaryDistanceIndex make_temporary_distance_index( #endif //Get the node - SnarlDistanceIndex::TemporaryDistanceIndex::TemporaryNodeRecord& temp_node_record = temp_index.temp_node_records.at(node_id - temp_index.min_node_id); + SnarlDistanceIndex::temp_record_ref_t node_index = make_pair(SnarlDistanceIndex::TEMP_NODE, node_id); + SnarlDistanceIndex::TemporaryDistanceIndex::TemporaryNodeRecord& temp_node_record = temp_index.get_node(node_ref); temp_node_record.reversed_in_parent = false; @@ -198,20 +199,21 @@ SnarlDistanceIndex::TemporaryDistanceIndex make_temporary_distance_index( for (nid_t next_id : reachable_nodes) { //For each node that this is connected to, check if we've already seen it and if we have, then //union this chain and that node's chain - SnarlDistanceIndex::TemporaryDistanceIndex::TemporaryNodeRecord& node_record = temp_index.temp_node_records[next_id-temp_index.min_node_id]; + SnarlDistanceIndex::temp_record_ref_t next_index = make_pair(SnarlDistanceIndex::TEMP_NODE, next_id); + SnarlDistanceIndex::TemporaryDistanceIndex::TemporaryNodeRecord& node_record = temp_index.get_node(next_ref); if (node_record.node_id != 0) { //If we've already seen this node, union it with the new one //If we can see it by walking out from this top-level chain, then it must also be a //top-level chain (or node pretending to be a chain) size_t other_i = node_record.parent.first == SnarlDistanceIndex::TEMP_CHAIN - ? temp_index.temp_chain_records[node_record.parent.second].root_snarl_index + ? temp_index.get_chain(node_record.parent).root_snarl_index : node_record.root_snarl_index; #ifdef debug_distance_indexing assert(other_i != std::numeric_limits::max()); #endif root_snarl_component_uf.union_groups(other_i, temp_node_record.root_snarl_index); //#ifdef debug_distance_indexing -// cerr << " Union this trivial with " << temp_index.temp_chain_records[node_record.parent.second].start_node_id << " " << temp_index.temp_chain_records[node_record.parent.second].end_node_id << endl; +// cerr << " Union this trivial with " << temp_index.get_chain(node_record.parent).start_node_id << " " << temp_index.get_chain(node_record.parent).end_node_id << endl; //#endif } else { new_component = false; @@ -225,7 +227,7 @@ SnarlDistanceIndex::TemporaryDistanceIndex make_temporary_distance_index( } else { //The last thing on the stack is the parent of this chain, which must be a snarl temp_node_record.parent = stack.back(); - auto& parent_snarl_record = temp_index.temp_snarl_records.at(temp_node_record.parent.second); + auto& parent_snarl_record = temp_index.get_snarl(temp_node_record.parent); temp_node_record.rank_in_parent = parent_snarl_record.children.size() + 2; parent_snarl_record.children.emplace_back(SnarlDistanceIndex::TEMP_NODE, node_id); } @@ -281,20 +283,21 @@ SnarlDistanceIndex::TemporaryDistanceIndex make_temporary_distance_index( for (nid_t next_id : reachable_nodes) { //For each node that this is connected to, check if we've already seen it and if we have, then //union this chain and that node's chain - SnarlDistanceIndex::TemporaryDistanceIndex::TemporaryNodeRecord& node_record = temp_index.temp_node_records[next_id-temp_index.min_node_id]; + SnarlDistanceIndex::temp_record_ref_t next_index = make_pair(SnarlDistanceIndex::TEMP_NODE, next_id); + SnarlDistanceIndex::TemporaryDistanceIndex::TemporaryNodeRecord& node_record = temp_index.get_node(next_ref); if (node_record.node_id != 0) { //If we've already seen this node, union it with the new one //If we can see it by walking out from this top-level chain, then it must also be a //top-level chain (or node pretending to be a chain) size_t other_i = node_record.parent.first == SnarlDistanceIndex::TEMP_CHAIN - ? temp_index.temp_chain_records[node_record.parent.second].root_snarl_index + ? temp_index.get_chain(node_record.parent).root_snarl_index : node_record.root_snarl_index; #ifdef debug_distance_indexing assert(other_i != std::numeric_limits::max()); #endif root_snarl_component_uf.union_groups(other_i, temp_chain_record.root_snarl_index); #ifdef debug_distance_indexing - cerr << " Union this chain with " << temp_index.temp_chain_records[node_record.parent.second].start_node_id << " " << temp_index.temp_chain_records[node_record.parent.second].end_node_id << endl; + cerr << " Union this chain with " << temp_index.get_chain(node_record.parent).start_node_id << " " << temp_index.get_chain(node_record.parent).end_node_id << endl; #endif } else { new_component = false; @@ -309,7 +312,7 @@ SnarlDistanceIndex::TemporaryDistanceIndex make_temporary_distance_index( } else { //The last thing on the stack is the parent of this chain, which must be a snarl temp_chain_record.parent = stack.back(); - auto& parent_snarl_record = temp_index.temp_snarl_records.at(temp_chain_record.parent.second); + auto& parent_snarl_record = temp_index.get_snarl(temp_chain_record.parent); temp_chain_record.rank_in_parent = parent_snarl_record.children.size() + 2; parent_snarl_record.children.emplace_back(chain_index); } @@ -346,13 +349,13 @@ SnarlDistanceIndex::TemporaryDistanceIndex make_temporary_distance_index( * parent chain * Also create a node record */ - pair snarl_index = stack.back(); + SnarlDistanceIndex::temp_record_ref_t snarl_index = stack.back(); stack.pop_back(); #ifdef debug_distance_indexing assert(snarl_index.first == SnarlDistanceIndex::TEMP_SNARL); assert(stack.back().first == SnarlDistanceIndex::TEMP_CHAIN); #endif - SnarlDistanceIndex::TemporaryDistanceIndex::TemporarySnarlRecord& temp_snarl_record = temp_index.temp_snarl_records[snarl_index.second]; + SnarlDistanceIndex::TemporaryDistanceIndex::TemporarySnarlRecord& temp_snarl_record = temp_index.get_snarl(snarl_index); nid_t node_id = graph->get_id(snarl_end_handle); //Record the end node in the snarl @@ -381,7 +384,7 @@ SnarlDistanceIndex::TemporaryDistanceIndex make_temporary_distance_index( assert(stack.back().first == SnarlDistanceIndex::TEMP_CHAIN); #endif temp_snarl_record.parent = stack.back(); - auto& temp_chain = temp_index.temp_chain_records.at(stack.back().second); + auto& temp_chain = temp_index.get_chain(stack.back()); temp_chain.children.emplace_back(SnarlDistanceIndex::TEMP_NODE, node_id); //Remove the snarl record @@ -395,7 +398,7 @@ SnarlDistanceIndex::TemporaryDistanceIndex make_temporary_distance_index( assert(stack.back().first == SnarlDistanceIndex::TEMP_CHAIN); #endif temp_snarl_record.parent = stack.back(); - auto& temp_chain = temp_index.temp_chain_records.at(stack.back().second); + auto& temp_chain = temp_index.get_chain(stack.back()); temp_chain.children.emplace_back(snarl_index); temp_chain.children.emplace_back(SnarlDistanceIndex::TEMP_NODE, node_id); @@ -411,7 +414,8 @@ SnarlDistanceIndex::TemporaryDistanceIndex make_temporary_distance_index( //} //Record the node itself. This gets done for the start of the chain, and ends of snarls - SnarlDistanceIndex::TemporaryDistanceIndex::TemporaryNodeRecord& temp_node_record = temp_index.temp_node_records.at(node_id-temp_index.min_node_id); + SnarlDistanceIndex::temp_record_ref_t node_index = make_pair(SnarlDistanceIndex::TEMP_NODE, node_id); + SnarlDistanceIndex::TemporaryDistanceIndex::TemporaryNodeRecord& temp_node_record = temp_index.get_node(node_ref); temp_node_record.node_id = node_id; temp_node_record.node_length = graph->get_length(snarl_end_handle); temp_node_record.reversed_in_parent = graph->get_is_reverse(snarl_end_handle); @@ -450,7 +454,7 @@ SnarlDistanceIndex::TemporaryDistanceIndex make_temporary_distance_index( for (size_t chain_i : root_snarl_indexes) { //For each chain component of this root-level snarl if (temp_index.root_snarl_components[chain_i].first == SnarlDistanceIndex::TEMP_CHAIN){ - SnarlDistanceIndex::TemporaryDistanceIndex::TemporaryChainRecord& temp_chain_record = temp_index.temp_chain_records[temp_index.root_snarl_components[chain_i].second]; + SnarlDistanceIndex::TemporaryDistanceIndex::TemporaryChainRecord& temp_chain_record = temp_index.get_chain(temp_index.root_snarl_components[chain_i]); temp_chain_record.parent = make_pair(SnarlDistanceIndex::TEMP_SNARL, temp_index.temp_snarl_records.size() - 1); temp_chain_record.rank_in_parent = temp_snarl_record.children.size(); temp_chain_record.reversed_in_parent = false; @@ -460,7 +464,7 @@ SnarlDistanceIndex::TemporaryDistanceIndex make_temporary_distance_index( #ifdef debug_distance_indexing assert(temp_index.root_snarl_components[chain_i].first == SnarlDistanceIndex::TEMP_NODE); #endif - SnarlDistanceIndex::TemporaryDistanceIndex::TemporaryNodeRecord& temp_node_record = temp_index.temp_node_records[temp_index.root_snarl_components[chain_i].second - temp_index.min_node_id]; + SnarlDistanceIndex::TemporaryDistanceIndex::TemporaryNodeRecord& temp_node_record = temp_index.get_node(temp_index.root_snarl_components[chain_i]); temp_node_record.parent = make_pair(SnarlDistanceIndex::TEMP_SNARL, temp_index.temp_snarl_records.size() - 1); temp_node_record.rank_in_parent = temp_snarl_record.children.size(); temp_node_record.reversed_in_parent = false; @@ -483,11 +487,11 @@ SnarlDistanceIndex::TemporaryDistanceIndex make_temporary_distance_index( cerr << "Filling in the distances in snarls" << endl; #endif for (int i = temp_index.temp_chain_records.size()-1 ; i >= 0 ; i--) { - - SnarlDistanceIndex::TemporaryDistanceIndex::TemporaryChainRecord& temp_chain_record = temp_index.temp_chain_records[i]; + SnarlDistanceIndex::temp_record_ref_t chain_index = make_pair(SnarlDistanceIndex::TEMP_CHAIN, i); + SnarlDistanceIndex::TemporaryDistanceIndex::TemporaryChainRecord& temp_chain_record = temp_index.get_chain(chain_ref); #ifdef debug_distance_indexing assert(!temp_chain_record.is_trivial); - cerr << " At " << (temp_chain_record.is_trivial ? " trivial " : "") << " chain " << temp_index.structure_start_end_as_string(make_pair(SnarlDistanceIndex::TEMP_CHAIN, i)) << endl; + cerr << " At " << (temp_chain_record.is_trivial ? " trivial " : "") << " chain " << temp_index.structure_start_end_as_string(chain_ref) << endl; #endif //Add the first values for the prefix sum and backwards loop vectors @@ -504,7 +508,7 @@ SnarlDistanceIndex::TemporaryDistanceIndex make_temporary_distance_index( size_t curr_component = 0; //which component of the chain are we in size_t last_node_length = 0; for (size_t chain_child_i = 0 ; chain_child_i < temp_chain_record.children.size() ; chain_child_i++ ){ - const pair& chain_child_index = temp_chain_record.children[chain_child_i]; + const SnarlDistanceIndex::temp_record_ref_t& chain_child_index = temp_chain_record.children[chain_child_i]; //Go through each of the children in the chain, skipping nodes //The snarl may be trivial, in which case don't fill in the distances #ifdef debug_distance_indexing @@ -517,7 +521,7 @@ SnarlDistanceIndex::TemporaryDistanceIndex make_temporary_distance_index( //all distances, then add distances to the chain that this is in //The parent chain will be the last thing in the stack SnarlDistanceIndex::TemporaryDistanceIndex::TemporarySnarlRecord& temp_snarl_record = - temp_index.temp_snarl_records.at(chain_child_index.second); + temp_index.get_snarl(chain_child_index); //Fill in this snarl's distances populate_snarl_index(temp_index, chain_child_index, size_limit, only_top_level_chain_distances, graph); @@ -565,7 +569,7 @@ SnarlDistanceIndex::TemporaryDistanceIndex make_temporary_distance_index( //If this is a node and the last thing was also a node, //then there was a trivial snarl SnarlDistanceIndex::TemporaryDistanceIndex::TemporaryNodeRecord& temp_node_record = - temp_index.temp_node_records.at(chain_child_index.second-temp_index.min_node_id); + temp_index.get_node(chain_child_index); //Check if there is a loop in this node //Snarls get counted as trivial if they contain no nodes but they might still have edges @@ -589,7 +593,7 @@ SnarlDistanceIndex::TemporaryDistanceIndex make_temporary_distance_index( } temp_chain_record.chain_components.emplace_back(curr_component); } - last_node_length = temp_index.temp_node_records.at(chain_child_index.second - temp_index.min_node_id).node_length; + last_node_length = temp_index.get_node(chain_child_index).node_length; //And update the chains max length temp_chain_record.max_length = SnarlDistanceIndex::sum(temp_chain_record.max_length, last_node_length); @@ -625,7 +629,7 @@ SnarlDistanceIndex::TemporaryDistanceIndex make_temporary_distance_index( //If this is a looping chain, then check the first snarl for a loop if (temp_chain_record.children.at(1).first == SnarlDistanceIndex::TEMP_SNARL) { - SnarlDistanceIndex::TemporaryDistanceIndex::TemporarySnarlRecord& temp_snarl_record = temp_index.temp_snarl_records.at(temp_chain_record.children.at(1).second); + SnarlDistanceIndex::TemporaryDistanceIndex::TemporarySnarlRecord& temp_snarl_record = temp_index.get_snarl(temp_chain_record.children.at(1)); temp_chain_record.forward_loops[temp_chain_record.forward_loops.size()-1] = temp_snarl_record.distance_start_start; } } @@ -636,7 +640,7 @@ SnarlDistanceIndex::TemporaryDistanceIndex make_temporary_distance_index( for (int j = (int)temp_chain_record.children.size() - 1 ; j >= 0 ; j--) { auto& child = temp_chain_record.children.at(j); if (child.first == SnarlDistanceIndex::TEMP_SNARL){ - SnarlDistanceIndex::TemporaryDistanceIndex::TemporarySnarlRecord& temp_snarl_record = temp_index.temp_snarl_records.at(child.second); + SnarlDistanceIndex::TemporaryDistanceIndex::TemporarySnarlRecord& temp_snarl_record = temp_index.get_snarl(child); if (temp_chain_record.chain_components.at(node_i) != temp_chain_record.chain_components.at(node_i+1) && temp_chain_record.chain_components.at(node_i+1) != 0){ //If this is a new chain component, then add the loop distance from the snarl @@ -655,7 +659,7 @@ SnarlDistanceIndex::TemporaryDistanceIndex make_temporary_distance_index( } else { if (last_node_length != 0) { SnarlDistanceIndex::TemporaryDistanceIndex::TemporaryNodeRecord& temp_node_record = - temp_index.temp_node_records.at(child.second-temp_index.min_node_id); + temp_index.get_node(child); //Check if there is a loop in this node @@ -672,7 +676,7 @@ SnarlDistanceIndex::TemporaryDistanceIndex make_temporary_distance_index( 2*last_node_length)); node_i--; } - last_node_length = temp_index.temp_node_records.at(child.second - temp_index.min_node_id).node_length; + last_node_length = temp_index.get_node(child).node_length; } } @@ -691,7 +695,7 @@ SnarlDistanceIndex::TemporaryDistanceIndex make_temporary_distance_index( for (size_t i = 1 ; i < temp_chain_record.children.size()-1 ; i++ ) { auto& child = temp_chain_record.children.at(i); if (child.first == SnarlDistanceIndex::TEMP_SNARL) { - SnarlDistanceIndex::TemporaryDistanceIndex::TemporarySnarlRecord& temp_snarl_record = temp_index.temp_snarl_records.at(child.second); + SnarlDistanceIndex::TemporaryDistanceIndex::TemporarySnarlRecord& temp_snarl_record = temp_index.get_snarl(child); size_t new_loop_distance = SnarlDistanceIndex::sum(SnarlDistanceIndex::sum( temp_chain_record.backward_loops.at(node_i-1), 2*temp_snarl_record.min_length), @@ -714,7 +718,7 @@ SnarlDistanceIndex::TemporaryDistanceIndex make_temporary_distance_index( temp_chain_record.backward_loops.at(node_i) = std::min(old_loop_distance,new_loop_distance); node_i++; } - last_node_length = temp_index.temp_node_records.at(child.second - temp_index.min_node_id).node_length; + last_node_length = temp_index.get_node(child).node_length; } } } @@ -728,7 +732,7 @@ SnarlDistanceIndex::TemporaryDistanceIndex make_temporary_distance_index( for (int j = (int)temp_chain_record.children.size() - 1 ; j >= 0 ; j--) { auto& child = temp_chain_record.children.at(j); if (child.first == SnarlDistanceIndex::TEMP_SNARL){ - SnarlDistanceIndex::TemporaryDistanceIndex::TemporarySnarlRecord& temp_snarl_record = temp_index.temp_snarl_records.at(child.second); + SnarlDistanceIndex::TemporaryDistanceIndex::TemporarySnarlRecord& temp_snarl_record = temp_index.get_snarl(child); size_t new_distance = SnarlDistanceIndex::sum(SnarlDistanceIndex::sum( temp_chain_record.forward_loops.at(node_i+1), 2* temp_snarl_record.min_length), @@ -750,7 +754,7 @@ SnarlDistanceIndex::TemporaryDistanceIndex make_temporary_distance_index( temp_chain_record.forward_loops.at(node_i) = std::min(old_distance, new_distance); node_i--; } - last_node_length = temp_index.temp_node_records.at(child.second - temp_index.min_node_id).node_length; + last_node_length = temp_index.get_node(child).node_length; } } } @@ -766,9 +770,9 @@ SnarlDistanceIndex::TemporaryDistanceIndex make_temporary_distance_index( #ifdef debug_distance_indexing cerr << "Filling in the distances in root snarls and distances along chains" << endl; #endif - for (pair& component_index : temp_index.components) { + for (SnarlDistanceIndex::temp_record_ref_t& component_index : temp_index.components) { if (component_index.first == SnarlDistanceIndex::TEMP_SNARL) { - SnarlDistanceIndex::TemporaryDistanceIndex::TemporarySnarlRecord& temp_snarl_record = temp_index.temp_snarl_records.at(component_index.second); + SnarlDistanceIndex::TemporaryDistanceIndex::TemporarySnarlRecord& temp_snarl_record = temp_index.get_snarl(component_index); populate_snarl_index(temp_index, component_index, size_limit, only_top_level_chain_distances, graph); temp_snarl_record.min_length = std::numeric_limits::max(); } @@ -781,7 +785,7 @@ SnarlDistanceIndex::TemporaryDistanceIndex make_temporary_distance_index( return temp_index; } -static void populate_distance_matrix_row(SnarlDistanceIndex::TemporaryDistanceIndex& temp_index, pair& snarl_index, SnarlDistanceIndex::TemporaryDistanceIndex::TemporarySnarlRecord& temp_snarl_record, const pair& start_index, const HandleGraph* graph, size_t start_rank, bool is_internal_node, size_t size_limit); +static void populate_distance_matrix_row(SnarlDistanceIndex::TemporaryDistanceIndex& temp_index, SnarlDistanceIndex::temp_record_ref_t& snarl_index, SnarlDistanceIndex::TemporaryDistanceIndex::TemporarySnarlRecord& temp_snarl_record, const SnarlDistanceIndex::temp_record_ref_t& start_index, const HandleGraph* graph, size_t start_rank, bool is_internal_node, size_t size_limit); /* Fills in required distance matrix rows for each child @@ -790,7 +794,7 @@ Fills in required distance matrix rows for each child - size_limit == 0: no distances in index, so no rows - Top-level chain distances only: ??? */ -static void populate_distance_matrix_if_needed(SnarlDistanceIndex::TemporaryDistanceIndex& temp_index, pair& snarl_index, SnarlDistanceIndex::TemporaryDistanceIndex::TemporarySnarlRecord& temp_snarl_record, vector>& all_children, const HandleGraph* graph, size_t size_limit, bool only_top_level_chain_distances); +static void populate_distance_matrix_if_needed(SnarlDistanceIndex::TemporaryDistanceIndex& temp_index, SnarlDistanceIndex::temp_record_ref_t& snarl_index, SnarlDistanceIndex::TemporaryDistanceIndex::TemporarySnarlRecord& temp_snarl_record, vector& all_children, const HandleGraph* graph, size_t size_limit, bool only_top_level_chain_distances); /* Does three things: @@ -798,7 +802,9 @@ Does three things: - Builds the hub labels - Stores labels in temp_snarl_record */ -static void populate_hub_labeling(SnarlDistanceIndex::TemporaryDistanceIndex& temp_index, pair& snarl_index, SnarlDistanceIndex::TemporaryDistanceIndex::TemporarySnarlRecord& temp_snarl_record, vector>& all_children, const HandleGraph* graph); +static void populate_hub_labeling(SnarlDistanceIndex::TemporaryDistanceIndex& temp_index, SnarlDistanceIndex::temp_record_ref_t& snarl_index, SnarlDistanceIndex::TemporaryDistanceIndex::TemporarySnarlRecord& temp_snarl_record, vector& all_children, const HandleGraph* graph); + +static std::variant< /*Fill in the snarl index. * The index will already know its boundaries and everything knows their relationships in the @@ -808,21 +814,21 @@ static void populate_hub_labeling(SnarlDistanceIndex::TemporaryDistanceIndex& te */ void populate_snarl_index( SnarlDistanceIndex::TemporaryDistanceIndex& temp_index, - pair snarl_index, size_t size_limit, + SnarlDistanceIndex::temp_record_ref_t snarl_index, size_t size_limit, bool only_top_level_chain_distances, const HandleGraph* graph) { #ifdef debug_distance_indexing cerr << "Getting the distances for snarl " << temp_index.structure_start_end_as_string(snarl_index) << endl; assert(snarl_index.first == SnarlDistanceIndex::TEMP_SNARL); #endif - SnarlDistanceIndex::TemporaryDistanceIndex::TemporarySnarlRecord& temp_snarl_record = temp_index.temp_snarl_records.at(snarl_index.second); + SnarlDistanceIndex::TemporaryDistanceIndex::TemporarySnarlRecord& temp_snarl_record = temp_index.get_snarl(snarl_index); temp_snarl_record.is_simple=true; /*Helper function to find the ancestor of a node that is a child of this snarl */ - auto get_ancestor_of_node = [&](pair curr_index, - pair ancestor_snarl_index) { + auto get_ancestor_of_node = [&](SnarlDistanceIndex::temp_record_ref_t curr_index, + SnarlDistanceIndex::temp_record_ref_t ancestor_snarl_index) { //This is a child that isn't a node, so it must be a chain if (curr_index.second == temp_snarl_record.start_node_id || @@ -831,11 +837,11 @@ void populate_snarl_index( } //Otherwise, walk up until we hit the current snarl - pair parent_index = temp_index.temp_node_records.at(curr_index.second-temp_index.min_node_id).parent; + SnarlDistanceIndex::temp_record_ref_t parent_index = temp_index.get_node(curr_index).parent; while (parent_index != ancestor_snarl_index) { curr_index=parent_index; - parent_index = parent_index.first == SnarlDistanceIndex::TEMP_SNARL ? temp_index.temp_snarl_records.at(parent_index.second).parent - : temp_index.temp_chain_records.at(parent_index.second).parent; + parent_index = parent_index.first == SnarlDistanceIndex::TEMP_SNARL ? temp_index.get_snarl(parent_index).parent + : temp_index.get_chain(parent_index).parent; #ifdef debug_distance_indexing assert(parent_index.first != SnarlDistanceIndex::TEMP_ROOT); #endif @@ -845,7 +851,7 @@ void populate_snarl_index( }; //TODO: Copying the list - vector> all_children = temp_snarl_record.children; + vector all_children = temp_snarl_record.children; /* * Do a topological sort of the children and re-assign ranks based on the sort @@ -861,19 +867,19 @@ void populate_snarl_index( // then flip is_reversed // Since we don't have distances in snarl ancestors yet, walk out the fronts of chains and see if // we hit the snarl start or end - pair current_index = snarl_index; + SnarlDistanceIndex::temp_record_ref_t current_index = snarl_index; while (current_index.first != SnarlDistanceIndex::TEMP_ROOT) { //Get the parent of the current index - pair parent_index = - current_index.first == SnarlDistanceIndex::TEMP_SNARL ? temp_index.temp_snarl_records.at(current_index.second).parent - : temp_index.temp_chain_records.at(current_index.second).parent; + SnarlDistanceIndex::temp_record_ref_t parent_index = + current_index.first == SnarlDistanceIndex::TEMP_SNARL ? temp_index.get_snarl(current_index).parent + : temp_index.get_chain(current_index).parent; if (parent_index.first == SnarlDistanceIndex::TEMP_SNARL) { //If the parent is a snarl, then walk out the front of the chain and see if it reaches the start of the ancestor snarl vector to_check; unordered_set seen; - to_check.emplace_back(graph->get_handle(temp_index.temp_chain_records[current_index.second].start_node_id, - !temp_index.temp_chain_records[current_index.second].start_node_rev)); + to_check.emplace_back(graph->get_handle(temp_index.get_chain(current_index).start_node_id, + !temp_index.get_chain(current_index).start_node_rev)); seen.emplace(to_check.back()); bool reaches_start = false; while (!to_check.empty()) { @@ -881,27 +887,27 @@ void populate_snarl_index( to_check.pop_back(); graph->follow_edges(current_handle, false, [&](const handle_t next_handle) { if (seen.count(next_handle) == 0) { - if (graph->get_id(next_handle) == temp_index.temp_snarl_records[parent_index.second].start_node_id) { + if (graph->get_id(next_handle) == temp_index.get_snarl(parent_index).start_node_id) { //If this reached the start node, then we consider the chain to be oriented forward // so we can stop reaches_start = true; //Stop iterating return false; - } else if (graph->get_id(next_handle) != temp_index.temp_snarl_records[parent_index.second].end_node_id) { + } else if (graph->get_id(next_handle) != temp_index.get_snarl(parent_index).end_node_id) { //If this isn't leaving the snarl, then continue traversing //We need to jump to the end of the current chain //First, find the temp_chain_record for the chain we just entered - pair next_index = + SnarlDistanceIndex::temp_record_ref_t next_index = get_ancestor_of_node(make_pair(SnarlDistanceIndex::TEMP_NODE, graph->get_id(next_handle)), parent_index); to_check.emplace_back( next_index.first == SnarlDistanceIndex::TEMP_NODE ? next_handle : - (graph->get_id(next_handle) == temp_index.temp_chain_records[next_index.second].start_node_id - ? graph->get_handle(temp_index.temp_chain_records[next_index.second].end_node_id, - temp_index.temp_chain_records[next_index.second].end_node_rev) - : graph->get_handle(temp_index.temp_chain_records[next_index.second].start_node_id, - !temp_index.temp_chain_records[next_index.second].start_node_rev))); + (graph->get_id(next_handle) == temp_index.get_chain(next_index).start_node_id + ? graph->get_handle(temp_index.get_chain(next_index).end_node_id, + temp_index.get_chain(next_index).end_node_rev) + : graph->get_handle(temp_index.get_chain(next_index).start_node_id, + !temp_index.get_chain(next_index).start_node_rev))); } seen.emplace(next_handle); @@ -979,18 +985,18 @@ void populate_snarl_index( //If the current child is the start bound, then get the start node pointing in current_graph_handle = topological_sort_start; } else { - pair current_index = all_children[current_child_index.first]; + SnarlDistanceIndex::temp_record_ref_t current_index = all_children[current_child_index.first]; if (current_index.first == SnarlDistanceIndex::TEMP_NODE) { //If the current child is a node, then get the node pointing in the correct direction current_graph_handle = graph->get_handle(current_index.second, current_child_index.second); } else if (current_child_index.second) { //If the current child is a chain, and we're traversing the chain backwards - current_graph_handle = graph->get_handle(temp_index.temp_chain_records[current_index.second].start_node_id, - !temp_index.temp_chain_records[current_index.second].start_node_rev); + current_graph_handle = graph->get_handle(temp_index.get_chain(current_index).start_node_id, + !temp_index.get_chain(current_index).start_node_rev); } else { //Otherwise, the current child is a chain and we're traversing the chain forwards - current_graph_handle = graph->get_handle(temp_index.temp_chain_records[current_index.second].end_node_id, - temp_index.temp_chain_records[current_index.second].end_node_rev); + current_graph_handle = graph->get_handle(temp_index.get_chain(current_index).end_node_id, + temp_index.get_chain(current_index).end_node_rev); } } @@ -1008,15 +1014,15 @@ void populate_snarl_index( //If it reaches anything unseen, then it can't be a source node //Get the index of next_handle - pair next_index = + SnarlDistanceIndex::temp_record_ref_t next_index = get_ancestor_of_node(make_pair(SnarlDistanceIndex::TEMP_NODE, graph->get_id(next_handle)), snarl_index); size_t next_rank = next_index.first == SnarlDistanceIndex::TEMP_NODE - ? temp_index.temp_node_records.at(next_index.second-temp_index.min_node_id).rank_in_parent - : temp_index.temp_chain_records[next_index.second].rank_in_parent; + ? temp_index.get_node(next_index).rank_in_parent + : temp_index.get_chain(next_index).rank_in_parent; assert(all_children[next_rank-2] == next_index); - bool next_rev = next_index.first == SnarlDistanceIndex::TEMP_NODE || temp_index.temp_chain_records[next_index.second].is_trivial + bool next_rev = next_index.first == SnarlDistanceIndex::TEMP_NODE || temp_index.get_chain(next_index).is_trivial ? graph->get_is_reverse(next_handle) - : graph->get_id(next_handle) == temp_index.temp_chain_records[next_index.second].end_node_id; + : graph->get_id(next_handle) == temp_index.get_chain(next_index).end_node_id; if (visited_nodes.count(make_pair(next_rank, next_rev)) != 0) { //If this is a loop, just skip it return true; @@ -1025,10 +1031,10 @@ void populate_snarl_index( //Get the handle from the child represented by next_handle going the other way handle_t reverse_handle = next_index.first == SnarlDistanceIndex::TEMP_NODE ? graph->get_handle(next_index.second, !next_rev) : - (next_rev ? graph->get_handle(temp_index.temp_chain_records[next_index.second].end_node_id, - temp_index.temp_chain_records[next_index.second].end_node_rev) - : graph->get_handle(temp_index.temp_chain_records[next_index.second].start_node_id, - !temp_index.temp_chain_records[next_index.second].start_node_rev)); + (next_rev ? graph->get_handle(temp_index.get_chain(next_index).end_node_id, + temp_index.get_chain(next_index).end_node_rev) + : graph->get_handle(temp_index.get_chain(next_index).start_node_id, + !temp_index.get_chain(next_index).start_node_rev)); //Does this have no unseen incoming edges? Check as we go through incoming edges bool is_source = true; @@ -1045,15 +1051,15 @@ void populate_snarl_index( return true; } //The index of the snarl's child that next_handle represents - pair incoming_index = + SnarlDistanceIndex::temp_record_ref_t incoming_index = get_ancestor_of_node(make_pair(SnarlDistanceIndex::TEMP_NODE, graph->get_id(incoming_handle)), snarl_index); size_t incoming_rank = incoming_index.first == SnarlDistanceIndex::TEMP_NODE - ? temp_index.temp_node_records.at(incoming_index.second-temp_index.min_node_id).rank_in_parent - : temp_index.temp_chain_records[incoming_index.second].rank_in_parent; + ? temp_index.get_node(incoming_index).rank_in_parent + : temp_index.get_chain(incoming_index).rank_in_parent; - bool incoming_rev = incoming_index.first == SnarlDistanceIndex::TEMP_NODE || temp_index.temp_chain_records[incoming_index.second].is_trivial + bool incoming_rev = incoming_index.first == SnarlDistanceIndex::TEMP_NODE || temp_index.get_chain(incoming_index).is_trivial ? graph->get_is_reverse(incoming_handle) - : graph->get_id(incoming_handle) == temp_index.temp_chain_records[incoming_index.second].end_node_id; + : graph->get_id(incoming_handle) == temp_index.get_chain(incoming_index).end_node_id; //subtract 2 to get the index from the rank assert(incoming_rank >= 2); incoming_rank-=2; @@ -1103,9 +1109,9 @@ void populate_snarl_index( for (size_t new_rank = 0 ; new_rank < topological_sort_order.size() ; new_rank++) { size_t old_rank = topological_sort_order[new_rank]; if (all_children[old_rank].first == SnarlDistanceIndex::TEMP_NODE) { - temp_index.temp_node_records.at(all_children[old_rank].second-temp_index.min_node_id).rank_in_parent = new_rank+2; + temp_index.get_node(all_children[old_rank]).rank_in_parent = new_rank+2; } else { - temp_index.temp_chain_records[all_children[old_rank].second].rank_in_parent = new_rank+2; + temp_index.get_chain(all_children[old_rank]).rank_in_parent = new_rank+2; } } } @@ -1143,7 +1149,7 @@ void populate_snarl_index( if (temp_snarl_record.is_simple) { for (size_t i = 0 ; i < temp_snarl_record.node_count ; i++) { //Get the index of the child - const pair& child_index = temp_snarl_record.children[i]; + const SnarlDistanceIndex::temp_record_ref_t& child_index = temp_snarl_record.children[i]; //Which is a node #ifdef debug_distance_indexing assert(child_index.first == SnarlDistanceIndex::TEMP_NODE); @@ -1151,7 +1157,7 @@ void populate_snarl_index( //And get the record SnarlDistanceIndex::TemporaryDistanceIndex::TemporaryNodeRecord& temp_node_record = - temp_index.temp_node_records[child_index.second-temp_index.min_node_id]; + temp_index.get_node(child_index); size_t rank =temp_node_record.rank_in_parent; @@ -1172,7 +1178,7 @@ void populate_snarl_index( temp_index.max_bits = std::max(temp_index.max_bits, 22 + SnarlDistanceIndex::bit_width(temp_snarl_record.children.size())); } -void populate_hub_labeling(SnarlDistanceIndex::TemporaryDistanceIndex& temp_index, pair& snarl_index, SnarlDistanceIndex::TemporaryDistanceIndex::TemporarySnarlRecord& temp_snarl_record, vector>& all_children, const HandleGraph* graph) { +void populate_hub_labeling(SnarlDistanceIndex::TemporaryDistanceIndex& temp_index, SnarlDistanceIndex::temp_record_ref_t& snarl_index, SnarlDistanceIndex::TemporaryDistanceIndex::TemporarySnarlRecord& temp_snarl_record, vector& all_children, const HandleGraph* graph) { CHOverlay ov = make_boost_graph(temp_index, snarl_index, temp_snarl_record, all_children, graph); make_contraction_hierarchy(ov); @@ -1183,7 +1189,7 @@ void populate_hub_labeling(SnarlDistanceIndex::TemporaryDistanceIndex& temp_inde temp_snarl_record.hub_labels = pack_labels(labels, labels_rev); } -void populate_distance_matrix_if_needed(SnarlDistanceIndex::TemporaryDistanceIndex& temp_index, pair& snarl_index, SnarlDistanceIndex::TemporaryDistanceIndex::TemporarySnarlRecord& temp_snarl_record, vector>& all_children, const HandleGraph* graph, size_t size_limit, bool only_top_level_chain_distances) { +void populate_distance_matrix_if_needed(SnarlDistanceIndex::TemporaryDistanceIndex& temp_index, SnarlDistanceIndex::temp_record_ref_t& snarl_index, SnarlDistanceIndex::TemporaryDistanceIndex::TemporarySnarlRecord& temp_snarl_record, vector& all_children, const HandleGraph* graph, size_t size_limit, bool only_top_level_chain_distances) { if (size_limit != 0 && !only_top_level_chain_distances) { //If we are saving distances //Reserve enough space to store all possible distances @@ -1192,7 +1198,7 @@ void populate_distance_matrix_if_needed(SnarlDistanceIndex::TemporaryDistanceInd : temp_snarl_record.node_count * temp_snarl_record.node_count); } while (!all_children.empty()) { - const pair start_index = std::move(all_children.back()); + const SnarlDistanceIndex::temp_record_ref_t start_index = std::move(all_children.back()); all_children.pop_back(); bool is_internal_node = false; @@ -1202,19 +1208,20 @@ void populate_distance_matrix_if_needed(SnarlDistanceIndex::TemporaryDistanceInd && start_index.second != temp_snarl_record.start_node_id && start_index.second != temp_snarl_record.end_node_id) || - (start_index.first == SnarlDistanceIndex::TEMP_CHAIN && temp_index.temp_chain_records.at(start_index.second).is_trivial)) { + (start_index.first == SnarlDistanceIndex::TEMP_CHAIN && temp_index.get_chain(start_index).is_trivial)) { //If this is an internal node is_internal_node = true; - nid_t node_id = start_index.first == SnarlDistanceIndex::TEMP_NODE ? start_index.second : temp_index.temp_chain_records.at(start_index.second).start_node_id; - size_t rank = start_index.first == SnarlDistanceIndex::TEMP_NODE ? temp_index.temp_node_records.at(start_index.second-temp_index.min_node_id).rank_in_parent - : temp_index.temp_chain_records.at(start_index.second).rank_in_parent; + nid_t node_id = start_index.first == SnarlDistanceIndex::TEMP_NODE ? start_index.second : temp_index.get_chain(start_index).start_node_id; + SnarlDistanceIndex::temp_record_ref_t node_index {SnarlDistanceIndex::TEMP_NODE, node_id}; + size_t rank = start_index.first == SnarlDistanceIndex::TEMP_NODE ? temp_index.get_node(start_index).rank_in_parent + : temp_index.get_chain(start_index).rank_in_parent; bool has_edges = false; graph->follow_edges(graph->get_handle(node_id, false), false, [&](const handle_t next_handle) { has_edges = true; }); if (!has_edges) { - temp_index.temp_node_records.at(node_id-temp_index.min_node_id).is_tip = true; + temp_index.get_node(node_ref).is_tip = true; temp_snarl_record.tippy_child_ranks.insert(rank); temp_snarl_record.is_simple=false; //It is a tip so this isn't simple snarl } @@ -1223,22 +1230,22 @@ void populate_distance_matrix_if_needed(SnarlDistanceIndex::TemporaryDistanceInd has_edges = true; }); if (!has_edges) { - temp_index.temp_node_records.at(node_id-temp_index.min_node_id).is_tip = true; + temp_index.temp_index.get_node(node_ref).is_tip = true; temp_snarl_record.tippy_child_ranks.insert(rank); temp_snarl_record.is_simple=false; //It is a tip so this isn't simple snarl } - } else if (start_index.first == SnarlDistanceIndex::TEMP_CHAIN && !temp_index.temp_chain_records.at(start_index.second).is_trivial) { + } else if (start_index.first == SnarlDistanceIndex::TEMP_CHAIN && !temp_index.get_chain(start_index).is_trivial) { //If this is an internal chain, then it isn't a simple snarl temp_snarl_record.is_simple=false; } bool start_is_tip = start_index.first == SnarlDistanceIndex::TEMP_NODE - ? temp_index.temp_node_records.at(start_index.second-temp_index.min_node_id).is_tip - : temp_index.temp_chain_records.at(start_index.second).is_tip; + ? temp_index.get_node(start_index).is_tip + : temp_index.get_chain(start_index).is_tip; size_t start_rank = start_index.first == SnarlDistanceIndex::TEMP_NODE - ? temp_index.temp_node_records.at(start_index.second-temp_index.min_node_id).rank_in_parent - : temp_index.temp_chain_records.at(start_index.second).rank_in_parent; + ? temp_index.get_node(start_index).rank_in_parent + : temp_index.get_chain(start_index).rank_in_parent; if (start_index.first == SnarlDistanceIndex::TEMP_NODE && start_index.second == temp_snarl_record.start_node_id) { @@ -1266,10 +1273,10 @@ void populate_distance_matrix_if_needed(SnarlDistanceIndex::TemporaryDistanceInd -void populate_distance_matrix_row(SnarlDistanceIndex::TemporaryDistanceIndex& temp_index, pair& snarl_index, SnarlDistanceIndex::TemporaryDistanceIndex::TemporarySnarlRecord& temp_snarl_record, const pair& start_index, const HandleGraph* graph, size_t start_rank, bool is_internal_node, size_t size_limit) { +void populate_distance_matrix_row(SnarlDistanceIndex::TemporaryDistanceIndex& temp_index, SnarlDistanceIndex::temp_record_ref_t& snarl_index, SnarlDistanceIndex::TemporaryDistanceIndex::TemporarySnarlRecord& temp_snarl_record, const SnarlDistanceIndex::temp_record_ref_t& start_index, const HandleGraph* graph, size_t start_rank, bool is_internal_node, size_t size_limit) { /*Helper function to find the ancestor of a node that is a child of this snarl */ - auto get_ancestor_of_node = [&](pair curr_index, - pair ancestor_snarl_index) { + auto get_ancestor_of_node = [&](SnarlDistanceIndex::temp_record_ref_t curr_index, + SnarlDistanceIndex::temp_record_ref_t ancestor_snarl_index) { //This is a child that isn't a node, so it must be a chain if (curr_index.second == temp_snarl_record.start_node_id || @@ -1278,11 +1285,11 @@ void populate_distance_matrix_row(SnarlDistanceIndex::TemporaryDistanceIndex& te } //Otherwise, walk up until we hit the current snarl - pair parent_index = temp_index.temp_node_records.at(curr_index.second-temp_index.min_node_id).parent; + SnarlDistanceIndex::temp_record_ref_t parent_index = temp_index.get_node(curr_index).parent; while (parent_index != ancestor_snarl_index) { curr_index=parent_index; - parent_index = parent_index.first == SnarlDistanceIndex::TEMP_SNARL ? temp_index.temp_snarl_records.at(parent_index.second).parent - : temp_index.temp_chain_records.at(parent_index.second).parent; + parent_index = parent_index.first == SnarlDistanceIndex::TEMP_SNARL ? temp_index.get_snarl(parent_index).parent + : temp_index.get_chain(parent_index).parent; #ifdef debug_distance_indexing assert(parent_index.first != SnarlDistanceIndex::TEMP_ROOT); #endif @@ -1313,7 +1320,7 @@ void populate_distance_matrix_row(SnarlDistanceIndex::TemporaryDistanceIndex& te //Define a NetgraphNode as the value for the priority queue: // , direction> - using NetgraphNode = pair, bool>>; + using NetgraphNode = pair>; auto cmp = [] (const NetgraphNode a, const NetgraphNode b) { return a.first > b.first; }; @@ -1321,7 +1328,7 @@ void populate_distance_matrix_row(SnarlDistanceIndex::TemporaryDistanceIndex& te //The priority queue of the next nodes to visit, ordered by the distance std::priority_queue, decltype(cmp)> queue(cmp); //The nodes we've already visited - unordered_set, bool>> visited_nodes; + unordered_set> visited_nodes; visited_nodes.reserve(temp_snarl_record.node_count * 2); //Start from the current start node @@ -1331,7 +1338,7 @@ void populate_distance_matrix_row(SnarlDistanceIndex::TemporaryDistanceIndex& te //Get the current node from the queue and pop it out of the queue size_t current_distance = queue.top().first; - pair current_index = queue.top().second.first; + SnarlDistanceIndex::temp_record_ref_t current_index = queue.top().second.first; bool current_rev = queue.top().second.second; if (visited_nodes.count(queue.top().second)) { queue.pop(); @@ -1346,10 +1353,10 @@ void populate_distance_matrix_row(SnarlDistanceIndex::TemporaryDistanceIndex& te //opposite side of the child chain handle_t current_end_handle = current_index.first == SnarlDistanceIndex::TEMP_NODE ? graph->get_handle(current_index.second, current_rev) : - (current_rev ? graph->get_handle(temp_index.temp_chain_records[current_index.second].start_node_id, - !temp_index.temp_chain_records[current_index.second].start_node_rev) - : graph->get_handle(temp_index.temp_chain_records[current_index.second].end_node_id, - temp_index.temp_chain_records[current_index.second].end_node_rev)); + (current_rev ? graph->get_handle(temp_index.get_chain(current_index).start_node_id, + !temp_index.get_chain(current_index).start_node_rev) + : graph->get_handle(temp_index.get_chain(current_index).end_node_id, + temp_index.get_chain(current_index).end_node_rev)); #ifdef debug_distance_indexing cerr << " at child " << temp_index.structure_start_end_as_string(current_index) << " going " @@ -1361,8 +1368,8 @@ void populate_distance_matrix_row(SnarlDistanceIndex::TemporaryDistanceIndex& te //If this loops onto the same node side then this isn't a simple snarl temp_snarl_record.is_simple = false; } else if ((current_index.first == SnarlDistanceIndex::TEMP_NODE ? current_index.second - : (current_rev ? temp_index.temp_chain_records[current_index.second].end_node_id - : temp_index.temp_chain_records[current_index.second].start_node_id)) + : (current_rev ? temp_index.get_chain(current_index).end_node_id + : temp_index.get_chain(current_index).start_node_id)) == graph->get_id(next_handle)){ //If this loops to the other end of the chain then this isn't a simple snarl temp_snarl_record.is_simple = false; @@ -1379,23 +1386,26 @@ void populate_distance_matrix_row(SnarlDistanceIndex::TemporaryDistanceIndex& te } reachable_node_count++; + + SnarlDistanceIndex::temp_record_ref_t next_node_index = make_pair(SnarlDistanceIndex::TEMP_NODE, graph->get_id(next_handle)); + //At each of the nodes reachable from the current one, fill in the distance from the start //node to the next node (current_distance). If this handle isn't leaving the snarl, //add the next nodes along with the distance to the end of the next node - auto& node_record = temp_index.temp_node_records.at(graph->get_id(next_handle)-temp_index.min_node_id); + auto& node_record = temp_index.get_node(next_node_ref); //The index of the snarl's child that next_handle represents - pair next_index = - get_ancestor_of_node(make_pair(SnarlDistanceIndex::TEMP_NODE, graph->get_id(next_handle)), snarl_index); + SnarlDistanceIndex::temp_record_ref_t next_index = + get_ancestor_of_node(next_node_ref, snarl_index); bool next_is_tip = start_index.first == SnarlDistanceIndex::TEMP_NODE - ? temp_index.temp_node_records.at(start_index.second-temp_index.min_node_id).is_tip - : temp_index.temp_chain_records.at(start_index.second).is_tip; + ? temp_index.get_node(start_index).is_tip + : temp_index.get_chain(start_index).is_tip; //The rank and orientation of next in the snarl size_t next_rank = next_index.first == SnarlDistanceIndex::TEMP_NODE ? node_record.rank_in_parent - : temp_index.temp_chain_records[next_index.second].rank_in_parent; + : temp_index.get_chain(next_index).rank_in_parent; if (next_index.first == SnarlDistanceIndex::TEMP_NODE && next_index.second == temp_snarl_record.start_node_id) { next_rank = 0; } else if (next_index.first == SnarlDistanceIndex::TEMP_NODE && next_index.second == temp_snarl_record.end_node_id) { @@ -1409,9 +1419,9 @@ void populate_distance_matrix_row(SnarlDistanceIndex::TemporaryDistanceIndex& te //else { // assert(next_rank != 0 && next_rank != 1); //} - bool next_rev = next_index.first == SnarlDistanceIndex::TEMP_NODE || temp_index.temp_chain_records[next_index.second].is_trivial + bool next_rev = next_index.first == SnarlDistanceIndex::TEMP_NODE || temp_index.get_chain(next_index).is_trivial ? graph->get_is_reverse(next_handle) - : graph->get_id(next_handle) == temp_index.temp_chain_records[next_index.second].end_node_id; + : graph->get_id(next_handle) == temp_index.get_chain(next_index).end_node_id; /**Record the distance **/ bool start_is_boundary = !temp_snarl_record.is_root_snarl && (start_rank == 0 || start_rank == 1); @@ -1448,7 +1458,7 @@ void populate_distance_matrix_row(SnarlDistanceIndex::TemporaryDistanceIndex& te //If start is a boundary node if (next_index.first == SnarlDistanceIndex::TEMP_NODE) { //Next is a node - auto& temp_node_record = temp_index.temp_node_records.at(next_index.second-temp_index.min_node_id); + auto& temp_node_record = temp_index.get_node(next_index); if (start_rank == 0 && !next_rev && temp_node_record.distance_left_start == std::numeric_limits::max()) { temp_node_record.distance_left_start = current_distance; @@ -1468,7 +1478,7 @@ void populate_distance_matrix_row(SnarlDistanceIndex::TemporaryDistanceIndex& te } } else { //Next is a chain - auto& temp_chain_record = temp_index.temp_chain_records.at(next_index.second); + auto& temp_chain_record = temp_index.get_chain(next_index); if (start_rank == 0 && !next_rev && temp_chain_record.distance_left_start == std::numeric_limits::max()) { temp_chain_record.distance_left_start = current_distance; @@ -1512,9 +1522,9 @@ void populate_distance_matrix_row(SnarlDistanceIndex::TemporaryDistanceIndex& te //If this isn't leaving the snarl, //then add the next node to the queue, along with the distance to traverse it size_t next_node_length = next_index.first == SnarlDistanceIndex::TEMP_NODE ? graph->get_length(next_handle) : - temp_index.temp_chain_records[next_index.second].min_length; + temp_index.get_chain(next_index).min_length; if (next_index.first == SnarlDistanceIndex::TEMP_CHAIN && - temp_index.temp_chain_records[next_index.second].chain_components.back() != 0) { + temp_index.get_chain(next_index).chain_components.back() != 0) { //If there are multiple components, then the chain is not start-end reachable so its length //is actually infinite next_node_length = std::numeric_limits::max(); @@ -1525,8 +1535,8 @@ void populate_distance_matrix_row(SnarlDistanceIndex::TemporaryDistanceIndex& te } } if (next_index.first == SnarlDistanceIndex::TEMP_CHAIN) { - size_t loop_distance = next_rev ? temp_index.temp_chain_records[next_index.second].backward_loops.back() - : temp_index.temp_chain_records[next_index.second].forward_loops.front(); + size_t loop_distance = next_rev ? temp_index.get_chain(next_index).backward_loops.back() + : temp_index.get_chain(next_index).forward_loops.front(); if (loop_distance != std::numeric_limits::max() && visited_nodes.count(make_pair(next_index, !next_rev)) == 0 && graph->get_id(next_handle) != temp_snarl_record.start_node_id && @@ -1554,23 +1564,23 @@ void populate_distance_matrix_row(SnarlDistanceIndex::TemporaryDistanceIndex& te if (start_rank != 0 && start_rank != 1) { size_t child_max_length = start_index.first == SnarlDistanceIndex::TEMP_NODE - ? temp_index.temp_node_records.at(start_index.second-temp_index.min_node_id).node_length - : temp_index.temp_chain_records.at(start_index.second).max_length; + ? temp_index.get_node(start_index).node_length + : temp_index.get_chain(start_index).max_length; //The distance through the whole snarl traversing this node forwards //(This might actually be traversing it backwards but it doesn't really matter) size_t dist_start_left = start_index.first == SnarlDistanceIndex::TEMP_NODE - ? temp_index.temp_node_records.at(start_index.second-temp_index.min_node_id).distance_left_start - : temp_index.temp_chain_records.at(start_index.second).distance_left_start; + ? temp_index.get_node(start_index).distance_left_start + : temp_index.get_chain(start_index).distance_left_start; size_t dist_end_right = start_index.first == SnarlDistanceIndex::TEMP_NODE - ? temp_index.temp_node_records.at(start_index.second-temp_index.min_node_id).distance_right_end - : temp_index.temp_chain_records.at(start_index.second).distance_right_end; + ? temp_index.get_node(start_index).distance_right_end + : temp_index.get_chain(start_index).distance_right_end; size_t dist_start_right = start_index.first == SnarlDistanceIndex::TEMP_NODE - ? temp_index.temp_node_records.at(start_index.second-temp_index.min_node_id).distance_right_start - : temp_index.temp_chain_records.at(start_index.second).distance_right_start; + ? temp_index.get_node(start_index).distance_right_start + : temp_index.get_chain(start_index).distance_right_start; size_t dist_end_left = start_index.first == SnarlDistanceIndex::TEMP_NODE - ? temp_index.temp_node_records.at(start_index.second-temp_index.min_node_id).distance_left_end - : temp_index.temp_chain_records.at(start_index.second).distance_left_end; + ? temp_index.get_node(start_index).distance_left_end + : temp_index.get_chain(start_index).distance_left_end; size_t snarl_length_fd = SnarlDistanceIndex::sum(SnarlDistanceIndex::sum( dist_start_left, dist_end_right),child_max_length); diff --git a/src/snarl_distance_index.hpp b/src/snarl_distance_index.hpp index af38cea819..e502b9aa12 100644 --- a/src/snarl_distance_index.hpp +++ b/src/snarl_distance_index.hpp @@ -37,7 +37,7 @@ void fill_in_distance_index(SnarlDistanceIndex* distance_index, const HandleGrap /// Fill in the temporary snarl record with distances void populate_snarl_index(SnarlDistanceIndex::TemporaryDistanceIndex& temp_index, - pair snarl_index, size_t size_limit, bool only_top_level_chain_distances, const HandleGraph* graph) ; + SnarlDistanceIndex::temp_record_ref_t snarl_index, size_t size_limit, bool only_top_level_chain_distances, const HandleGraph* graph) ; SnarlDistanceIndex::TemporaryDistanceIndex make_temporary_distance_index(const HandleGraph* graph, const HandleGraphSnarlFinder* snarl_finder, size_t size_limit, bool only_top_level_chain_distances); From 788224d40ea2673025d63a515de8148ad3a6eb6a Mon Sep 17 00:00:00 2001 From: Adam Novak Date: Fri, 23 Jan 2026 18:34:14 -0500 Subject: [PATCH 05/77] Use accessors so we can build the Tiny oversized snarl test index and get a wrong answer --- deps/libbdsg | 2 +- src/snarl_distance_index.cpp | 22 ++++++++++------------ 2 files changed, 11 insertions(+), 13 deletions(-) diff --git a/deps/libbdsg b/deps/libbdsg index 0065e9717c..5edfaa08d9 160000 --- a/deps/libbdsg +++ b/deps/libbdsg @@ -1 +1 @@ -Subproject commit 0065e9717c949c878427fe39aacb977fc2508e2d +Subproject commit 5edfaa08d9632108bfb57b3b1d50393b7bfa12b6 diff --git a/src/snarl_distance_index.cpp b/src/snarl_distance_index.cpp index 9d65aefc90..072d3a6900 100644 --- a/src/snarl_distance_index.cpp +++ b/src/snarl_distance_index.cpp @@ -159,7 +159,7 @@ SnarlDistanceIndex::TemporaryDistanceIndex make_temporary_distance_index( //Get the node SnarlDistanceIndex::temp_record_ref_t node_index = make_pair(SnarlDistanceIndex::TEMP_NODE, node_id); - SnarlDistanceIndex::TemporaryDistanceIndex::TemporaryNodeRecord& temp_node_record = temp_index.get_node(node_ref); + SnarlDistanceIndex::TemporaryDistanceIndex::TemporaryNodeRecord& temp_node_record = temp_index.get_node(node_index); temp_node_record.reversed_in_parent = false; @@ -200,7 +200,7 @@ SnarlDistanceIndex::TemporaryDistanceIndex make_temporary_distance_index( //For each node that this is connected to, check if we've already seen it and if we have, then //union this chain and that node's chain SnarlDistanceIndex::temp_record_ref_t next_index = make_pair(SnarlDistanceIndex::TEMP_NODE, next_id); - SnarlDistanceIndex::TemporaryDistanceIndex::TemporaryNodeRecord& node_record = temp_index.get_node(next_ref); + SnarlDistanceIndex::TemporaryDistanceIndex::TemporaryNodeRecord& node_record = temp_index.get_node(next_index); if (node_record.node_id != 0) { //If we've already seen this node, union it with the new one //If we can see it by walking out from this top-level chain, then it must also be a @@ -284,7 +284,7 @@ SnarlDistanceIndex::TemporaryDistanceIndex make_temporary_distance_index( //For each node that this is connected to, check if we've already seen it and if we have, then //union this chain and that node's chain SnarlDistanceIndex::temp_record_ref_t next_index = make_pair(SnarlDistanceIndex::TEMP_NODE, next_id); - SnarlDistanceIndex::TemporaryDistanceIndex::TemporaryNodeRecord& node_record = temp_index.get_node(next_ref); + SnarlDistanceIndex::TemporaryDistanceIndex::TemporaryNodeRecord& node_record = temp_index.get_node(next_index); if (node_record.node_id != 0) { //If we've already seen this node, union it with the new one //If we can see it by walking out from this top-level chain, then it must also be a @@ -415,7 +415,7 @@ SnarlDistanceIndex::TemporaryDistanceIndex make_temporary_distance_index( //Record the node itself. This gets done for the start of the chain, and ends of snarls SnarlDistanceIndex::temp_record_ref_t node_index = make_pair(SnarlDistanceIndex::TEMP_NODE, node_id); - SnarlDistanceIndex::TemporaryDistanceIndex::TemporaryNodeRecord& temp_node_record = temp_index.get_node(node_ref); + SnarlDistanceIndex::TemporaryDistanceIndex::TemporaryNodeRecord& temp_node_record = temp_index.get_node(node_index); temp_node_record.node_id = node_id; temp_node_record.node_length = graph->get_length(snarl_end_handle); temp_node_record.reversed_in_parent = graph->get_is_reverse(snarl_end_handle); @@ -488,10 +488,10 @@ SnarlDistanceIndex::TemporaryDistanceIndex make_temporary_distance_index( #endif for (int i = temp_index.temp_chain_records.size()-1 ; i >= 0 ; i--) { SnarlDistanceIndex::temp_record_ref_t chain_index = make_pair(SnarlDistanceIndex::TEMP_CHAIN, i); - SnarlDistanceIndex::TemporaryDistanceIndex::TemporaryChainRecord& temp_chain_record = temp_index.get_chain(chain_ref); + SnarlDistanceIndex::TemporaryDistanceIndex::TemporaryChainRecord& temp_chain_record = temp_index.get_chain(chain_index); #ifdef debug_distance_indexing assert(!temp_chain_record.is_trivial); - cerr << " At " << (temp_chain_record.is_trivial ? " trivial " : "") << " chain " << temp_index.structure_start_end_as_string(chain_ref) << endl; + cerr << " At " << (temp_chain_record.is_trivial ? " trivial " : "") << " chain " << temp_index.structure_start_end_as_string(chain_index) << endl; #endif //Add the first values for the prefix sum and backwards loop vectors @@ -804,8 +804,6 @@ Does three things: */ static void populate_hub_labeling(SnarlDistanceIndex::TemporaryDistanceIndex& temp_index, SnarlDistanceIndex::temp_record_ref_t& snarl_index, SnarlDistanceIndex::TemporaryDistanceIndex::TemporarySnarlRecord& temp_snarl_record, vector& all_children, const HandleGraph* graph); -static std::variant< - /*Fill in the snarl index. * The index will already know its boundaries and everything knows their relationships in the * snarl tree. This needs to fill in the distances and the ranks of children in the snarl @@ -1221,7 +1219,7 @@ void populate_distance_matrix_if_needed(SnarlDistanceIndex::TemporaryDistanceInd has_edges = true; }); if (!has_edges) { - temp_index.get_node(node_ref).is_tip = true; + temp_index.get_node(node_index).is_tip = true; temp_snarl_record.tippy_child_ranks.insert(rank); temp_snarl_record.is_simple=false; //It is a tip so this isn't simple snarl } @@ -1230,7 +1228,7 @@ void populate_distance_matrix_if_needed(SnarlDistanceIndex::TemporaryDistanceInd has_edges = true; }); if (!has_edges) { - temp_index.temp_index.get_node(node_ref).is_tip = true; + temp_index.get_node(node_index).is_tip = true; temp_snarl_record.tippy_child_ranks.insert(rank); temp_snarl_record.is_simple=false; //It is a tip so this isn't simple snarl } @@ -1392,11 +1390,11 @@ void populate_distance_matrix_row(SnarlDistanceIndex::TemporaryDistanceIndex& te //At each of the nodes reachable from the current one, fill in the distance from the start //node to the next node (current_distance). If this handle isn't leaving the snarl, //add the next nodes along with the distance to the end of the next node - auto& node_record = temp_index.get_node(next_node_ref); + auto& node_record = temp_index.get_node(next_node_index); //The index of the snarl's child that next_handle represents SnarlDistanceIndex::temp_record_ref_t next_index = - get_ancestor_of_node(next_node_ref, snarl_index); + get_ancestor_of_node(next_node_index, snarl_index); bool next_is_tip = start_index.first == SnarlDistanceIndex::TEMP_NODE ? temp_index.get_node(start_index).is_tip From 4985468c22c8880ee567d58f827c2f43029a6dc0 Mon Sep 17 00:00:00 2001 From: Adam Novak Date: Mon, 26 Jan 2026 17:58:51 -0500 Subject: [PATCH 06/77] Try dumping hub label data for debugging --- deps/libbdsg | 2 +- src/snarl_distance_index.cpp | 10 +++++++++- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/deps/libbdsg b/deps/libbdsg index 5edfaa08d9..b9ba5007d2 160000 --- a/deps/libbdsg +++ b/deps/libbdsg @@ -1 +1 @@ -Subproject commit 5edfaa08d9632108bfb57b3b1d50393b7bfa12b6 +Subproject commit b9ba5007d2bd9762f93d0adb32bc5a4540ec3bb4 diff --git a/src/snarl_distance_index.cpp b/src/snarl_distance_index.cpp index 072d3a6900..389fb50fd6 100644 --- a/src/snarl_distance_index.cpp +++ b/src/snarl_distance_index.cpp @@ -1183,8 +1183,16 @@ void populate_hub_labeling(SnarlDistanceIndex::TemporaryDistanceIndex& temp_inde vector> labels; labels.resize(num_vertices(ov)); vector> labels_rev; labels_rev.resize(num_vertices(ov)); create_labels(labels, labels_rev, ov); - //TODO: Put labels in temp_snarl_record + // Put labels in temp_snarl_record temp_snarl_record.hub_labels = pack_labels(labels, labels_rev); + std::cerr << "Hub labels as packed: " + for (size_t i = 0; i < temp_snarl_record.hub_labels.size(); i++) { + if (i > 0) { + std::cerr << " | "; + } + std::cerr << temp_snarl_record.hub_labels[i]; + } + std::cerr << std::endl; } void populate_distance_matrix_if_needed(SnarlDistanceIndex::TemporaryDistanceIndex& temp_index, SnarlDistanceIndex::temp_record_ref_t& snarl_index, SnarlDistanceIndex::TemporaryDistanceIndex::TemporarySnarlRecord& temp_snarl_record, vector& all_children, const HandleGraph* graph, size_t size_limit, bool only_top_level_chain_distances) { From 86e4e3163521847a233a56d03759a13bc7da5999 Mon Sep 17 00:00:00 2001 From: Adam Novak Date: Tue, 27 Jan 2026 18:29:02 -0500 Subject: [PATCH 07/77] Add synthetic Boost graph dumping code, and missing semicolon, and libbdsg that makes labels that can fit --- deps/libbdsg | 2 +- src/snarl_distance_index.cpp | 38 +++++++++++++++++++++++++++++++++++- 2 files changed, 38 insertions(+), 2 deletions(-) diff --git a/deps/libbdsg b/deps/libbdsg index b9ba5007d2..9ae76d4007 160000 --- a/deps/libbdsg +++ b/deps/libbdsg @@ -1 +1 @@ -Subproject commit b9ba5007d2bd9762f93d0adb32bc5a4540ec3bb4 +Subproject commit 9ae76d4007bd07a2d14ce34a47d97871d6d93318 diff --git a/src/snarl_distance_index.cpp b/src/snarl_distance_index.cpp index 389fb50fd6..dbdf4ed0ad 100644 --- a/src/snarl_distance_index.cpp +++ b/src/snarl_distance_index.cpp @@ -1178,14 +1178,50 @@ void populate_snarl_index( void populate_hub_labeling(SnarlDistanceIndex::TemporaryDistanceIndex& temp_index, SnarlDistanceIndex::temp_record_ref_t& snarl_index, SnarlDistanceIndex::TemporaryDistanceIndex::TemporarySnarlRecord& temp_snarl_record, vector& all_children, const HandleGraph* graph) { CHOverlay ov = make_boost_graph(temp_index, snarl_index, temp_snarl_record, all_children, graph); + + // Dump CHOverlay graph to stderr for debugging + std::cerr << "=== CHOverlay Graph Dump ===" << std::endl; + std::cerr << "Vertices: " << num_vertices(ov) << ", Edges: " << num_edges(ov) << std::endl; + std::cerr << "--- Nodes ---" << std::endl; + for (auto v : boost::make_iterator_range(vertices(ov))) { + const NodeProp& np = ov[v]; + std::cerr << "Node " << v << ": seqlen=" << np.seqlen + << " max_out=" << np.max_out + << " contracted_neighbors=" << np.contracted_neighbors + << " level=" << np.level + << " arc_cover=" << np.arc_cover + << " contracted=" << (np.contracted ? "true" : "false") + << " new_id=" << np.new_id << std::endl; + } + std::cerr << "--- Edges ---" << std::endl; + for (auto e : boost::make_iterator_range(edges(ov))) { + const EdgeProp& ep = ov[e]; + std::cerr << "Edge " << source(e, ov) << " -> " << target(e, ov) + << ": contracted=" << (ep.contracted ? "true" : "false") + << " weight=" << ep.weight + << " arc_cover=" << ep.arc_cover + << " ori=" << (ep.ori ? "true" : "false") << std::endl; + } + std::cerr << "=== End CHOverlay Dump ===" << std::endl; + make_contraction_hierarchy(ov); vector> labels; labels.resize(num_vertices(ov)); vector> labels_rev; labels_rev.resize(num_vertices(ov)); create_labels(labels, labels_rev, ov); + std::cerr << "Hub labels unpacked:" << std::endl; + for (const auto& node_list : {labels, labels_rev}) { + std::cerr << "Labels for all nodes:" << std::endl; + for (size_t i = 0; i < node_list.size(); i++) { + std::cerr << "\tLabels for rank " << i << ":" << std::endl; + for (const HubRecord& label : node_list[i]) { + std::cerr << "\t\tHub: " << label.hub << " Dist: " << label.dist << std::endl; + } + } + } // Put labels in temp_snarl_record temp_snarl_record.hub_labels = pack_labels(labels, labels_rev); - std::cerr << "Hub labels as packed: " + std::cerr << "Hub labels as packed: "; for (size_t i = 0; i < temp_snarl_record.hub_labels.size(); i++) { if (i > 0) { std::cerr << " | "; From e84e65794e7f954f744fe49a1bc12fc105087307 Mon Sep 17 00:00:00 2001 From: Adam Novak Date: Wed, 28 Jan 2026 17:58:44 -0500 Subject: [PATCH 08/77] Use libbdsg with slightly more implemented hub labeling integration --- deps/libbdsg | 2 +- src/snarl_distance_index.cpp | 8 +++++++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/deps/libbdsg b/deps/libbdsg index 1d50a682f4..fb774f20fe 160000 --- a/deps/libbdsg +++ b/deps/libbdsg @@ -1 +1 @@ -Subproject commit 1d50a682f49709c70d5897258e04057ef8194433 +Subproject commit fb774f20fe382a3c70155d47f8d2bdc0144e51cd diff --git a/src/snarl_distance_index.cpp b/src/snarl_distance_index.cpp index a0df698120..43807b27e6 100644 --- a/src/snarl_distance_index.cpp +++ b/src/snarl_distance_index.cpp @@ -2,6 +2,7 @@ //#define debug_snarl_traversal //#define debug_distances //#define debug_subgraph +//#define debug_hub_label_storage #include "snarl_distance_index.hpp" @@ -1041,7 +1042,7 @@ void populate_snarl_index( */ - //Add the start and end nodes to the list of children so that we include them in the traversal + // Add the start and end nodes to the list of children so that we include them in the traversal. if (!temp_snarl_record.is_root_snarl) { all_children.emplace_back(SnarlDistanceIndex::TEMP_NODE, temp_snarl_record.start_node_id); all_children.emplace_back(SnarlDistanceIndex::TEMP_NODE, temp_snarl_record.end_node_id); @@ -1131,6 +1132,7 @@ void populate_hub_labeling(SnarlDistanceIndex::TemporaryDistanceIndex& temp_inde vector> labels; labels.resize(num_vertices(ov)); vector> labels_rev; labels_rev.resize(num_vertices(ov)); create_labels(labels, labels_rev, ov); +#ifdef debug_hub_label_storage std::cerr << "Hub labels unpacked:" << std::endl; for (const auto& node_list : {labels, labels_rev}) { std::cerr << "Labels for all nodes:" << std::endl; @@ -1141,8 +1143,11 @@ void populate_hub_labeling(SnarlDistanceIndex::TemporaryDistanceIndex& temp_inde } } } +#endif + // Put labels in temp_snarl_record temp_snarl_record.hub_labels = pack_labels(labels, labels_rev); +#ifdef debug_hub_label_storage std::cerr << "Hub labels as packed: "; for (size_t i = 0; i < temp_snarl_record.hub_labels.size(); i++) { if (i > 0) { @@ -1151,6 +1156,7 @@ void populate_hub_labeling(SnarlDistanceIndex::TemporaryDistanceIndex& temp_inde std::cerr << temp_snarl_record.hub_labels[i]; } std::cerr << std::endl; +#endif } void populate_distance_matrix_if_needed(SnarlDistanceIndex::TemporaryDistanceIndex& temp_index, SnarlDistanceIndex::temp_record_ref_t& snarl_index, SnarlDistanceIndex::TemporaryDistanceIndex::TemporarySnarlRecord& temp_snarl_record, vector& all_children, const HandleGraph* graph, size_t size_limit, bool only_top_level_chain_distances) { From 4f31496653fa98d9b88979e8f627af0a1719450c Mon Sep 17 00:00:00 2001 From: Adam Novak Date: Thu, 29 Jan 2026 08:57:28 -0800 Subject: [PATCH 09/77] Make sure NodeProp fields are not used before initialization --- deps/libbdsg | 2 +- src/snarl_distance_index.cpp | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/deps/libbdsg b/deps/libbdsg index fb774f20fe..a5c20a8ee1 160000 --- a/deps/libbdsg +++ b/deps/libbdsg @@ -1 +1 @@ -Subproject commit fb774f20fe382a3c70155d47f8d2bdc0144e51cd +Subproject commit a5c20a8ee157f117dde5715e514b94e8e8edc815 diff --git a/src/snarl_distance_index.cpp b/src/snarl_distance_index.cpp index 43807b27e6..73540b7179 100644 --- a/src/snarl_distance_index.cpp +++ b/src/snarl_distance_index.cpp @@ -1114,7 +1114,8 @@ void populate_hub_labeling(SnarlDistanceIndex::TemporaryDistanceIndex& temp_inde << " level=" << np.level << " arc_cover=" << np.arc_cover << " contracted=" << (np.contracted ? "true" : "false") - << " new_id=" << np.new_id << std::endl; + // Skip new_id since it is not initialized until make_contraction_hierarchy is run. + << std::endl; } std::cerr << "--- Edges ---" << std::endl; for (auto e : boost::make_iterator_range(edges(ov))) { From 232a5898133fd8563fb8d7a4125215ddebdd7108 Mon Sep 17 00:00:00 2001 From: Adam Novak Date: Thu, 29 Jan 2026 11:00:10 -0800 Subject: [PATCH 10/77] Stop trying to look up removed trivial snarls --- deps/libbdsg | 2 +- src/snarl_distance_index.cpp | 30 +++++++++++++----------------- 2 files changed, 14 insertions(+), 18 deletions(-) diff --git a/deps/libbdsg b/deps/libbdsg index a5c20a8ee1..9a7e4c32c0 160000 --- a/deps/libbdsg +++ b/deps/libbdsg @@ -1 +1 @@ -Subproject commit a5c20a8ee157f117dde5715e514b94e8e8edc815 +Subproject commit 9a7e4c32c0b3c3ddc5fc65df1b5c29a959e68b20 diff --git a/src/snarl_distance_index.cpp b/src/snarl_distance_index.cpp index 73540b7179..85f49b5b32 100644 --- a/src/snarl_distance_index.cpp +++ b/src/snarl_distance_index.cpp @@ -381,6 +381,11 @@ SnarlDistanceIndex::TemporaryDistanceIndex make_temporary_distance_index( //This is a trivial snarl temp_snarl_record.is_trivial = true; +#ifdef debug_distance_indexing + cerr << " Ending and forgetting trivial snarl " << temp_index.structure_start_end_as_string(snarl_index) + << endl << " that is a child of " << temp_index.structure_start_end_as_string(temp_snarl_record.parent) << endl; +#endif + //Add the end node to the chain #ifdef debug_distance_indexing assert(stack.back().first == SnarlDistanceIndex::TEMP_CHAIN); @@ -389,13 +394,20 @@ SnarlDistanceIndex::TemporaryDistanceIndex make_temporary_distance_index( auto& temp_chain = temp_index.get_chain(stack.back()); temp_chain.children.emplace_back(SnarlDistanceIndex::TEMP_NODE, node_id); - //Remove the snarl record + //Remove the snarl record. + //This invalidates snarl_index!!! #ifdef debug_distance_indexing assert(temp_index.temp_snarl_records.size() == snarl_index.second+1); #endif temp_index.temp_snarl_records.pop_back(); } else { //This is the child of a chain + +#ifdef debug_distance_indexing + cerr << " Ending new snarl " << temp_index.structure_start_end_as_string(snarl_index) + << endl << " that is a child of " << temp_index.structure_start_end_as_string(temp_snarl_record.parent) << endl; +#endif + #ifdef debug_distance_indexing assert(stack.back().first == SnarlDistanceIndex::TEMP_CHAIN); #endif @@ -405,15 +417,6 @@ SnarlDistanceIndex::TemporaryDistanceIndex make_temporary_distance_index( temp_chain.children.emplace_back(SnarlDistanceIndex::TEMP_NODE, node_id); } - //Record the snarl as a child of its chain - //if (stack.empty()) { - // assert(false); - // //TODO: The snarl should always be the child of a chain - // //If this was the last thing on the stack, then this was a root - // //TODO: I'm not sure if this would get put into a chain or not - // temp_snarl_record.parent = make_pair(SnarlDistanceIndex::TEMP_ROOT, 0); - // temp_index.components.emplace_back(snarl_index); - //} //Record the node itself. This gets done for the start of the chain, and ends of snarls SnarlDistanceIndex::temp_record_ref_t node_index = make_pair(SnarlDistanceIndex::TEMP_NODE, node_id); @@ -422,13 +425,6 @@ SnarlDistanceIndex::TemporaryDistanceIndex make_temporary_distance_index( temp_node_record.node_length = graph->get_length(snarl_end_handle); temp_node_record.reversed_in_parent = graph->get_is_reverse(snarl_end_handle); temp_node_record.parent = stack.back(); - - - -#ifdef debug_distance_indexing - cerr << " Ending new snarl " << temp_index.structure_start_end_as_string(snarl_index) - << endl << " that is a child of " << temp_index.structure_start_end_as_string(temp_snarl_record.parent) << endl; -#endif }); /* From 30e392af7d9eb64244d55e821db756f370f4cbf1 Mon Sep 17 00:00:00 2001 From: Adam Novak Date: Thu, 29 Jan 2026 14:41:21 -0800 Subject: [PATCH 11/77] Add the debugging to subgraph finding that I needed to fix ChainRecord asserts --- deps/libbdsg | 2 +- src/snarl_distance_index.cpp | 13 ++++++++++--- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/deps/libbdsg b/deps/libbdsg index 9a7e4c32c0..41ccdf3410 160000 --- a/deps/libbdsg +++ b/deps/libbdsg @@ -1 +1 @@ -Subproject commit 9a7e4c32c0b3c3ddc5fc65df1b5c29a959e68b20 +Subproject commit 41ccdf3410a3c4232d9d8aa99190997704fb3e15 diff --git a/src/snarl_distance_index.cpp b/src/snarl_distance_index.cpp index 85f49b5b32..2467d7e661 100644 --- a/src/snarl_distance_index.cpp +++ b/src/snarl_distance_index.cpp @@ -1,7 +1,7 @@ #define debug_distance_indexing //#define debug_snarl_traversal //#define debug_distances -//#define debug_subgraph +#define debug_subgraph //#define debug_hub_label_storage #include "snarl_distance_index.hpp" @@ -1650,6 +1650,7 @@ cerr << "Start positon: "<< start_pos << endl; while (!distance_index.is_root(parent)) { #ifdef debug_subgraph cerr << "At child " << distance_index.net_handle_as_string(current_net) << " with distances " << current_distance_left << " " << current_distance_right << endl; + cerr << "Parent is " << distance_index.net_handle_as_string(parent) << " at offset " << SnarlDistanceIndex::get_record_offset(parent) << endl; #endif size_t max_parent_length = distance_index.maximum_length(parent); @@ -1677,7 +1678,7 @@ cerr << "Start positon: "<< start_pos << endl; if (distance_index.is_snarl(parent)) { //If this is the child of a snarl, then just traverse from the end of the node #ifdef debug_subgraph -cerr << "Start search in parent " << distance_index.net_handle_as_string(parent); + cerr << "Start search in parent " << distance_index.net_handle_as_string(parent); #endif if (current_distance_left != std::numeric_limits::max() ){ //If we can go left @@ -1724,7 +1725,7 @@ cerr << "Start search in parent " << distance_index.net_handle_as_string(parent) #endif } else { #ifdef debug_subgraph -cerr << "Start search along parent chain " << distance_index.net_handle_as_string(parent); + cerr << "Start search along parent chain " << distance_index.net_handle_as_string(parent); #endif //If this is the child of a chain, then traverse along the chain if (current_distance_left != std::numeric_limits::max()) { @@ -1739,6 +1740,9 @@ cerr << "Start search along parent chain " << distance_index.net_handle_as_strin subgraph_in_distance_range_walk_graph(super_graph, min_distance, max_distance, subgraph, search_start_nodes, seen_nodes, traversal_start); return; } else if (distance_index.is_snarl(parent)){ +#ifdef debug_subgraph + cerr << "Parent is a snarl of handle type " << SnarlDistanceIndex::get_handle_type(parent) << " at offset " << SnarlDistanceIndex::get_record_offset(parent) << endl; +#endif //TODO: This might be overkill. It prevents us from adding nodes that shouldn't be in the subgraph, but might be too slow //If we don't check the other direction, go through the loop and add everything whose distance is lower than the minimum //to seen_nodes @@ -1769,6 +1773,9 @@ cerr << "Start search along parent chain " << distance_index.net_handle_as_strin }); } } else if (distance_index.is_chain(parent)) { +#ifdef debug_subgraph + cerr << "Parent is a chain of handle type " << SnarlDistanceIndex::get_handle_type(parent) << " at offset " << SnarlDistanceIndex::get_record_offset(parent) << endl; +#endif //TODO: This is probably also overkill - walk a chain if there is a viable loop size_t distance_loop_right = distance_index.distance_in_parent(parent, current_net, current_net, super_graph, max_distance); size_t distance_loop_left = distance_index.distance_in_parent(parent, distance_index.flip(current_net), distance_index.flip(current_net), super_graph, max_distance); From 9639b68c42ef82d69ad5b48277ad1b54fe7ef817 Mon Sep 17 00:00:00 2001 From: Adam Novak Date: Mon, 2 Feb 2026 13:47:15 -0800 Subject: [PATCH 12/77] Stop trying to interpret the root as a chain in debug prints --- deps/libbdsg | 2 +- src/snarl_distance_index.cpp | 6 +++++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/deps/libbdsg b/deps/libbdsg index 41ccdf3410..a3948d447c 160000 --- a/deps/libbdsg +++ b/deps/libbdsg @@ -1 +1 @@ -Subproject commit 41ccdf3410a3c4232d9d8aa99190997704fb3e15 +Subproject commit a3948d447c18ed0515445e2f2e15dc66dc2cdde1 diff --git a/src/snarl_distance_index.cpp b/src/snarl_distance_index.cpp index 2467d7e661..053f762e72 100644 --- a/src/snarl_distance_index.cpp +++ b/src/snarl_distance_index.cpp @@ -299,7 +299,11 @@ SnarlDistanceIndex::TemporaryDistanceIndex make_temporary_distance_index( #endif root_snarl_component_uf.union_groups(other_i, temp_chain_record.root_snarl_index); #ifdef debug_distance_indexing - cerr << " Union this chain with " << temp_index.get_chain(node_record.parent).start_node_id << " " << temp_index.get_chain(node_record.parent).end_node_id << endl; + if (node_record.parent.first == SnarlDistanceIndex::TEMP_CHAIN) { + cerr << " Union this chain with " << temp_index.get_chain(node_record.parent).start_node_id << " " << temp_index.get_chain(node_record.parent).end_node_id << endl; + } else { + cerr << " Union this chain with root " << node_record.root_snarl_index << endl; + } #endif } else { new_component = false; From c0db406872ffc771e65f0731197440285f07d998 Mon Sep 17 00:00:00 2001 From: Adam Novak Date: Mon, 2 Feb 2026 14:41:47 -0800 Subject: [PATCH 13/77] Turn off debugging after passing existing snarl distance index tests --- deps/libbdsg | 2 +- src/snarl_distance_index.cpp | 7 +++++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/deps/libbdsg b/deps/libbdsg index a3948d447c..8a85c23624 160000 --- a/deps/libbdsg +++ b/deps/libbdsg @@ -1 +1 @@ -Subproject commit a3948d447c18ed0515445e2f2e15dc66dc2cdde1 +Subproject commit 8a85c23624317919589a16ba7c433d23b8519c0c diff --git a/src/snarl_distance_index.cpp b/src/snarl_distance_index.cpp index 053f762e72..2eafabb500 100644 --- a/src/snarl_distance_index.cpp +++ b/src/snarl_distance_index.cpp @@ -1,7 +1,8 @@ -#define debug_distance_indexing +//#define debug_distance_indexing //#define debug_snarl_traversal //#define debug_distances -#define debug_subgraph +//#define debug_subgraph +//#define debug_hub_label_build //#define debug_hub_label_storage #include "snarl_distance_index.hpp" @@ -1102,6 +1103,7 @@ void populate_snarl_index( void populate_hub_labeling(SnarlDistanceIndex::TemporaryDistanceIndex& temp_index, SnarlDistanceIndex::temp_record_ref_t& snarl_index, SnarlDistanceIndex::TemporaryDistanceIndex::TemporarySnarlRecord& temp_snarl_record, vector& all_children, const HandleGraph* graph) { CHOverlay ov = make_boost_graph(temp_index, snarl_index, temp_snarl_record, all_children, graph); +#ifdef debug_hub_label_build // Dump CHOverlay graph to stderr for debugging std::cerr << "=== CHOverlay Graph Dump ===" << std::endl; std::cerr << "Vertices: " << num_vertices(ov) << ", Edges: " << num_edges(ov) << std::endl; @@ -1127,6 +1129,7 @@ void populate_hub_labeling(SnarlDistanceIndex::TemporaryDistanceIndex& temp_inde << " ori=" << (ep.ori ? "true" : "false") << std::endl; } std::cerr << "=== End CHOverlay Dump ===" << std::endl; +#endif make_contraction_hierarchy(ov); From 163764fff769e60d723451af0d468c2918b156c4 Mon Sep 17 00:00:00 2001 From: Adam Novak Date: Tue, 10 Feb 2026 13:56:24 -0500 Subject: [PATCH 14/77] Make randomized graph test actually exercise oversized snarls sometimes --- src/unittest/snarl_distance_index.cpp | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/src/unittest/snarl_distance_index.cpp b/src/unittest/snarl_distance_index.cpp index 93a86342b2..093c45d0aa 100644 --- a/src/unittest/snarl_distance_index.cpp +++ b/src/unittest/snarl_distance_index.cpp @@ -22,7 +22,7 @@ #include #include "xg.hpp" -//#define debug +#define debug namespace vg { namespace unittest { @@ -7260,6 +7260,9 @@ namespace vg { } + // TODO: This test case doesn't do anything (runs 0 iterations). + // When I tell it to actually run iterations, it fails. + // Has it ever worked? TEST_CASE("random test subgraph", "[snarl_distance][snarl_distance_subgraph]") { int64_t min = 20; int64_t max = 50; @@ -7459,11 +7462,11 @@ namespace vg { uniform_int_distribution variant_count_dist(1, bases/30); size_t variant_count = variant_count_dist(generator); - uniform_int_distribution snarl_size_limit_dist(500, 1000); + uniform_int_distribution snarl_size_limit_dist(2, 1000); size_t size_limit = snarl_size_limit_dist(generator); #ifdef debug - cerr << repeat << ": Do graph of " << bases << " bp with ~" << variant_bases << " bp large variant length and " << variant_count << " events" << endl; + cerr << repeat << ": Do graph of " << bases << " bp with ~" << variant_bases << " bp large variant length and " << variant_count << " events with size limit " << size_limit << endl; #endif VG graph; @@ -7697,7 +7700,7 @@ namespace vg { size_t size_limit = snarl_size_limit_dist(generator); #ifdef debug - cerr << repeat << ": Do graph of " << bases << " bp with ~" << variant_bases << " bp large variant length and " << variant_count << " events" << endl; + cerr << repeat << ": Do graph of " << bases << " bp with ~" << variant_bases << " bp large variant length and " << variant_count << " events with size limit " << size_limit << endl; #endif VG graph; From ddce5f4c8bddb3ed4bc8af1e450fee8766b9bf3f Mon Sep 17 00:00:00 2001 From: Adam Novak Date: Tue, 10 Feb 2026 14:09:20 -0500 Subject: [PATCH 15/77] Add function for loading a handlegraph from JSON --- src/unittest/support/json.cpp | 24 ++++++++++++++++++++++++ src/unittest/support/json.hpp | 21 +++++++++++++++++++++ src/unittest/support/random_graph.hpp | 13 +++++++++---- 3 files changed, 54 insertions(+), 4 deletions(-) create mode 100644 src/unittest/support/json.cpp create mode 100644 src/unittest/support/json.hpp diff --git a/src/unittest/support/json.cpp b/src/unittest/support/json.cpp new file mode 100644 index 0000000000..8f6bdf8426 --- /dev/null +++ b/src/unittest/support/json.cpp @@ -0,0 +1,24 @@ +#include "json.hpp" + +#include "vg/io/json2pb.h" +#include "vg.hpp" + +namespace vg { +namespace unittest { + +std::unique_ptr json_to_graph(const std::string& json) { + // Load into a Protobuf object + Graph source; + json2pb(source, json.c_str(), json.size()); + + // Make a HandleGraph that knows how to load from Protobuf + std::unique_ptr to_return = new vg::VG(); + + // Load it from Protobuf + to_return->extend(source); + return to_return; +} + + +} +} diff --git a/src/unittest/support/json.hpp b/src/unittest/support/json.hpp new file mode 100644 index 0000000000..63e51834a0 --- /dev/null +++ b/src/unittest/support/json.hpp @@ -0,0 +1,21 @@ +#ifndef VG_UNITTEST_JSON_HPP_INCLUDED +#define VG_UNITTEST_JSON_HPP_INCLUDED +/** \file json.hpp + * Utilities for working with JSON data in test cases. + */ + +#include "handle.hpp" +#include + + +namespace vg { +namespace unittest { + +/// Create a handlegraph from vg Protobuf JSON. +std::unique_ptr json_to_graph(const std::string& json); + + +} +} + +#endif diff --git a/src/unittest/support/random_graph.hpp b/src/unittest/support/random_graph.hpp index 7597beeab9..e3e812d265 100644 --- a/src/unittest/support/random_graph.hpp +++ b/src/unittest/support/random_graph.hpp @@ -1,11 +1,16 @@ +#ifndef VG_UNITTEST_RANDOM_GRAPH_HPP_INCLUDED +#define VG_UNITTEST_RANDOM_GRAPH_HPP_INCLUDED +/** \file random_graph.hpp + * Utilities for randomizing graphs for test cases. + */ + + #include "handle.hpp" #include -#ifndef VG_UNITTEST_RANDOM_GRAPH_HPP_INCLUDED -#define VG_UNITTEST_RANDOM_GRAPH_HPP_INCLUDED -namespace vg{ -namespace unittest{ +namespace vg { +namespace unittest { /// Create a random graph by adding variation to a sequence of length seq_size /// variant_len is the mean length of a larger variation and variant_count From e56353f3aca57717da8d23c2ca06603edfd5a7dc Mon Sep 17 00:00:00 2001 From: Adam Novak Date: Tue, 10 Feb 2026 15:12:05 -0500 Subject: [PATCH 16/77] Allow cactus-ifying all handle graphs --- src/cactus.cpp | 4 ++-- src/cactus.hpp | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/cactus.cpp b/src/cactus.cpp index 6179663968..49eab63294 100644 --- a/src/cactus.cpp +++ b/src/cactus.cpp @@ -999,8 +999,8 @@ VG cactus_to_vg(stCactusGraph* cactus_graph) { return vg_graph; } -VG cactusify(VG& graph) { - if (graph.size() == 0) { +VG cactusify(const PathHandleGraph& graph) { + if (graph.get_node_count() == 0) { return VG(); } auto parts = handle_graph_to_cactus(graph, unordered_set()); diff --git a/src/cactus.hpp b/src/cactus.hpp index 36d53f2fab..21cfd8ebc7 100644 --- a/src/cactus.hpp +++ b/src/cactus.hpp @@ -46,7 +46,7 @@ VG cactus_to_vg(stCactusGraph* cactus_graph); // Convert vg into vg formatted cactus representation // Input graph must be sorted! -VG cactusify(VG& graph); +VG cactusify(const PathHandleGraph& graph); } From 4f66c258f68e7c2ff056b20d92e518abd93c6020 Mon Sep 17 00:00:00 2001 From: Adam Novak Date: Tue, 10 Feb 2026 15:12:43 -0500 Subject: [PATCH 17/77] Add synthetic fix for actually populating the unique_ptr right --- src/unittest/support/json.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/unittest/support/json.cpp b/src/unittest/support/json.cpp index 8f6bdf8426..a3954ab746 100644 --- a/src/unittest/support/json.cpp +++ b/src/unittest/support/json.cpp @@ -10,9 +10,9 @@ std::unique_ptr json_to_graph(const std::string& // Load into a Protobuf object Graph source; json2pb(source, json.c_str(), json.size()); - + // Make a HandleGraph that knows how to load from Protobuf - std::unique_ptr to_return = new vg::VG(); + auto to_return = std::make_unique(); // Load it from Protobuf to_return->extend(source); From 5436d73117a8c5c30e57dd359ced458a8ee6f693 Mon Sep 17 00:00:00 2001 From: Adam Novak Date: Tue, 10 Feb 2026 15:14:24 -0500 Subject: [PATCH 18/77] Commit partial synthetic refactor to use new JSON load method --- src/unittest/cactus.cpp | 21 ++-- src/unittest/chunker.cpp | 9 +- src/unittest/copy_graph.cpp | 89 +++++++--------- src/unittest/dijkstra.cpp | 14 ++- src/unittest/gbwt_extender.cpp | 6 +- src/unittest/genotypekit.cpp | 57 +++++----- src/unittest/genotyper.cpp | 9 +- src/unittest/haplotypes.cpp | 26 ++--- src/unittest/indexed_vg.cpp | 1 + src/unittest/mapper.cpp | 32 ++---- src/unittest/minimizer_mapper.cpp | 23 ++--- src/unittest/multipath_alignment_graph.cpp | 7 +- src/unittest/multipath_mapper.cpp | 25 ++--- src/unittest/path_component_index.cpp | 9 +- src/unittest/phase_unfolder.cpp | 33 ++---- src/unittest/readfilter.cpp | 6 +- src/unittest/snarl_distance_index.cpp | 37 ++----- src/unittest/snarls.cpp | 49 +++------ src/unittest/variant_adder.cpp | 115 +++++++-------------- src/unittest/vg.cpp | 7 +- src/unittest/vg_algorithms.cpp | 35 ++----- src/unittest/vpkg.cpp | 41 ++------ src/unittest/xdrop_aligner.cpp | 7 +- src/unittest/xg.cpp | 29 ++---- 24 files changed, 230 insertions(+), 457 deletions(-) diff --git a/src/unittest/cactus.cpp b/src/unittest/cactus.cpp index 7447ee247d..89a2f95cb8 100644 --- a/src/unittest/cactus.cpp +++ b/src/unittest/cactus.cpp @@ -8,6 +8,7 @@ #include "vg/io/json2pb.h" #include "../cactus.hpp" #include "catch.hpp" +#include "support/json.hpp" namespace vg { namespace unittest { @@ -15,8 +16,6 @@ using namespace std; TEST_CASE("We can convert a two-tailed graph to Cactus", "[cactus]") { - VG graph; - string graph_json = R"( {"node":[{"sequence":"GT","id":7575}, {"sequence":"TGTTAACAGCACAACATTTA","id":7580}, @@ -26,19 +25,15 @@ TEST_CASE("We can convert a two-tailed graph to Cactus", "[cactus]") { {"from":7575,"to":7576}]} )"; - Graph g; - json2pb(g, graph_json.c_str(), graph_json.size()); - graph.extend(g); + auto graph = json_to_graph(graph_json); - // Make sure we can make a Cactus graph and get something out. - auto cactusified = cactusify(graph); + // Make sure we can make a Cactus graph and get something out. + auto cactusified = cactusify(*graph); REQUIRE(cactusified.is_valid()); } TEST_CASE("We can convert a hairpin graph to Cactus", "[cactus]") { - VG graph; - // Here's a graph where only the left side of node 2 is dangling, and the right side of node 1 has a self loop. string graph_json = R"( {"node": [{"sequence": "A", "id": 1}, @@ -47,12 +42,10 @@ TEST_CASE("We can convert a hairpin graph to Cactus", "[cactus]") { {"from": 1, "to": 1, "to_end": true}]} )"; - Graph g; - json2pb(g, graph_json.c_str(), graph_json.size()); - graph.extend(g); + auto graph = json_to_graph(graph_json); - // Make sure we can make a Cactus graph and get something out. - auto cactusified = cactusify(graph); + // Make sure we can make a Cactus graph and get something out. + auto cactusified = cactusify(*graph); REQUIRE(cactusified.is_valid()); } diff --git a/src/unittest/chunker.cpp b/src/unittest/chunker.cpp index 24f7d3b645..81b136fe91 100644 --- a/src/unittest/chunker.cpp +++ b/src/unittest/chunker.cpp @@ -7,6 +7,7 @@ #include "vg.hpp" #include "xg.hpp" #include "path.hpp" +#include "support/json.hpp" namespace vg { namespace unittest { @@ -83,13 +84,9 @@ TEST_CASE("basic graph chunking", "[chunk]") { )"; - // Load it into Protobuf - Graph chunk; - json2pb(chunk, graph_json.c_str(), graph_json.size()); - - // Pass it over to XG + // Load it and pass it over to XG xg::XG index; - index.from_path_handle_graph(VG(chunk)); + index.from_path_handle_graph(*json_to_graph(graph_json)); PathChunker chunker(&index); diff --git a/src/unittest/copy_graph.cpp b/src/unittest/copy_graph.cpp index 581b683130..6d83c36fda 100644 --- a/src/unittest/copy_graph.cpp +++ b/src/unittest/copy_graph.cpp @@ -2,6 +2,7 @@ #include "../handle.hpp" #include "../vg.hpp" #include "xg.hpp" +#include "support/json.hpp" #include "bdsg/packed_graph.hpp" #include "bdsg/hash_graph.hpp" @@ -53,14 +54,13 @@ namespace vg { ] } )"; - Graph proto_graph; - json2pb(proto_graph, graph_json.c_str(), graph_json.size()); - + auto graph_ptr = json_to_graph(graph_json); + xg::XG xg; - xg.from_path_handle_graph(VG(proto_graph)); + xg.from_path_handle_graph(*graph_ptr); VG vg; handlealgs::copy_handle_graph(&xg, &vg); - + REQUIRE(xg.get_node_count() == 1); REQUIRE(vg.get_node_count() == 1); } @@ -72,14 +72,13 @@ namespace vg { ] } )"; - Graph proto_graph; - json2pb(proto_graph, graph_json.c_str(), graph_json.size()); - + auto graph_ptr = json_to_graph(graph_json); + xg::XG xg; - xg.from_path_handle_graph(VG(proto_graph)); + xg.from_path_handle_graph(*graph_ptr); bdsg::PackedGraph pg; handlealgs::copy_handle_graph(&xg, &pg); - + REQUIRE(xg.get_node_count() == 1); REQUIRE(pg.get_node_count() == 1); } @@ -91,11 +90,10 @@ namespace vg { ] } )"; - Graph proto_graph; - json2pb(proto_graph, graph_json.c_str(), graph_json.size()); - + auto graph_ptr = json_to_graph(graph_json); + xg::XG xg; - xg.from_path_handle_graph(VG(proto_graph)); + xg.from_path_handle_graph(*graph_ptr); bdsg::HashGraph hg; handlealgs::copy_handle_graph(&xg, &hg); @@ -120,11 +118,10 @@ namespace vg { ] } )"; - Graph proto_graph; - json2pb(proto_graph, graph_json.c_str(), graph_json.size()); - + auto graph_ptr = json_to_graph(graph_json); + xg::XG xg; - xg.from_path_handle_graph(VG(proto_graph)); + xg.from_path_handle_graph(*graph_ptr); VG vg; handlealgs::copy_handle_graph(&xg, &vg); @@ -151,11 +148,10 @@ namespace vg { ] } )"; - Graph proto_graph; - json2pb(proto_graph, graph_json.c_str(), graph_json.size()); - + auto graph_ptr = json_to_graph(graph_json); + xg::XG xg; - xg.from_path_handle_graph(VG(proto_graph)); + xg.from_path_handle_graph(*graph_ptr); bdsg::PackedGraph pg; handlealgs::copy_handle_graph(&xg, &pg); @@ -194,11 +190,10 @@ namespace vg { ] } )"; - Graph proto_graph; - json2pb(proto_graph, graph_json.c_str(), graph_json.size()); - + auto graph_ptr = json_to_graph(graph_json); + xg::XG xg; - xg.from_path_handle_graph(VG(proto_graph)); + xg.from_path_handle_graph(*graph_ptr); bdsg::HashGraph hg; handlealgs::copy_handle_graph(&xg, &hg); @@ -239,11 +234,10 @@ namespace vg { ] } )"; - Graph proto_graph; - json2pb(proto_graph, graph_json.c_str(), graph_json.size()); - + auto graph_ptr = json_to_graph(graph_json); + xg::XG xg; - xg.from_path_handle_graph(VG(proto_graph)); + xg.from_path_handle_graph(*graph_ptr); VG vg; handlealgs::copy_handle_graph(&xg, &vg); @@ -274,11 +268,10 @@ namespace vg { ] } )"; - Graph proto_graph; - json2pb(proto_graph, graph_json.c_str(), graph_json.size()); - + auto graph_ptr = json_to_graph(graph_json); + xg::XG xg; - xg.from_path_handle_graph(VG(proto_graph)); + xg.from_path_handle_graph(*graph_ptr); bdsg::PackedGraph pg; handlealgs::copy_handle_graph(&xg, &pg); @@ -321,11 +314,10 @@ namespace vg { ] } )"; - Graph proto_graph; - json2pb(proto_graph, graph_json.c_str(), graph_json.size()); - + auto graph_ptr = json_to_graph(graph_json); + xg::XG xg; - xg.from_path_handle_graph(VG(proto_graph)); + xg.from_path_handle_graph(*graph_ptr); bdsg::HashGraph hg; handlealgs::copy_handle_graph(&xg, &hg); @@ -382,11 +374,10 @@ namespace vg { ] } )"; - Graph proto_graph; - json2pb(proto_graph, graph_json.c_str(), graph_json.size()); - + auto graph_ptr = json_to_graph(graph_json); + xg::XG xg; - xg.from_path_handle_graph(VG(proto_graph)); + xg.from_path_handle_graph(*graph_ptr); VG vg; handlealgs::copy_path_handle_graph(&xg, &vg); @@ -444,11 +435,10 @@ namespace vg { ] } )"; - Graph proto_graph; - json2pb(proto_graph, graph_json.c_str(), graph_json.size()); - + auto graph_ptr = json_to_graph(graph_json); + xg::XG xg; - xg.from_path_handle_graph(VG(proto_graph)); + xg.from_path_handle_graph(*graph_ptr); bdsg::PackedGraph pg; handlealgs::copy_path_handle_graph(&xg, &pg); @@ -521,11 +511,10 @@ namespace vg { ] } )"; - Graph proto_graph; - json2pb(proto_graph, graph_json.c_str(), graph_json.size()); - + auto graph_ptr = json_to_graph(graph_json); + xg::XG xg; - xg.from_path_handle_graph(VG(proto_graph)); + xg.from_path_handle_graph(*graph_ptr); bdsg::HashGraph hg; handlealgs::copy_path_handle_graph(&xg, &hg); diff --git a/src/unittest/dijkstra.cpp b/src/unittest/dijkstra.cpp index 2608567153..e3d09743b7 100644 --- a/src/unittest/dijkstra.cpp +++ b/src/unittest/dijkstra.cpp @@ -9,6 +9,7 @@ #include "vg/io/json2pb.h" #include "../vg.hpp" #include "catch.hpp" +#include "support/json.hpp" #include @@ -128,30 +129,27 @@ TEST_CASE("Dijkstra search works on a particular problem graph", "[dijkstra][alg {"node":[{"sequence":"A","id":"2454530"},{"sequence":"AGTGCTGGAGAGGATGTGGAGAAATAGGAAC","id":"2454529"},{"sequence":"C","id":"2454532"},{"sequence":"TTTTACACTGTTGGTGGGACTGTAAA","id":"2454533"},{"sequence":"A","id":"2454527"},{"sequence":"C","id":"2454528"},{"sequence":"G","id":"2454531"},{"sequence":"C","id":"2454534"},{"sequence":"T","id":"2454535"},{"sequence":"GGGTAATAA","id":"2454526"},{"sequence":"TAGTTCAACCATTGTGGAAGACTGTGGCAATT","id":"2454536"}],"edge":[{"from":"2454530","to":"2454532"},{"from":"2454530","to":"2454533"},{"from":"2454529","to":"2454530"},{"from":"2454529","to":"2454531"},{"from":"2454532","to":"2454533"},{"from":"2454533","to":"2454534"},{"from":"2454533","to":"2454535"},{"from":"2454527","to":"2454529"},{"from":"2454528","to":"2454529"},{"from":"2454531","to":"2454532"},{"from":"2454531","to":"2454533"},{"from":"2454534","to":"2454536"},{"from":"2454535","to":"2454536"},{"from":"2454526","to":"2454527"},{"from":"2454526","to":"2454528"}],"path":[{"name":"21","mapping":[{"position":{"node_id":"2454526"},"edit":[{"from_length":9,"to_length":9}],"rank":"3049077"},{"position":{"node_id":"2454528"},"edit":[{"from_length":1,"to_length":1}],"rank":"3049078"},{"position":{"node_id":"2454529"},"edit":[{"from_length":31,"to_length":31}],"rank":"3049079"},{"position":{"node_id":"2454531"},"edit":[{"from_length":1,"to_length":1}],"rank":"3049080"},{"position":{"node_id":"2454532"},"edit":[{"from_length":1,"to_length":1}],"rank":"3049081"},{"position":{"node_id":"2454533"},"edit":[{"from_length":26,"to_length":26}],"rank":"3049082"},{"position":{"node_id":"2454535"},"edit":[{"from_length":1,"to_length":1}],"rank":"3049083"},{"position":{"node_id":"2454536"},"edit":[{"from_length":32,"to_length":32}],"rank":"3049084"}]}]} )"; - Graph g; - json2pb(g, graph_json); - // Wrap the graph in a HandleGraph - VG graph(g); + auto graph = json_to_graph(graph_json); // Decide where to start - handle_t start = graph.get_handle(2454536, true); + handle_t start = graph->get_handle(2454536, true); // Track what we reach and at what distance unordered_map seen; - handlealgs::dijkstra(&graph, start, [&](const handle_t& reached, size_t distance) { + handlealgs::dijkstra(graph.get(), start, [&](const handle_t& reached, size_t distance) { seen[reached] = distance; return true; }); - REQUIRE(seen.size() == graph.get_node_count()); + REQUIRE(seen.size() == graph->get_node_count()); } TEST_CASE( "Shortest path through chain with loop", "[dijkstra][algorithms]" ) { - VG graph; + bdsg::HashGraph graph; handle_t n1 = graph.create_handle("GCA"); handle_t n2 = graph.create_handle("T"); diff --git a/src/unittest/gbwt_extender.cpp b/src/unittest/gbwt_extender.cpp index d04a225fdb..b8b24710dd 100644 --- a/src/unittest/gbwt_extender.cpp +++ b/src/unittest/gbwt_extender.cpp @@ -8,6 +8,7 @@ #include "vg/io/json2pb.h" #include "../utility.hpp" #include "../vg.hpp" +#include "support/json.hpp" #include @@ -90,9 +91,8 @@ gbwt::GBWT build_gbwt_index() { // Build a GBWTGraph using the provided GBWT index. gbwtgraph::GBWTGraph build_gbwt_graph(const gbwt::GBWT& gbwt_index) { - Graph graph; - json2pb(graph, gapless_extender_graph.c_str(), gapless_extender_graph.size()); - VG vg_graph(graph); + auto vg_graph_ptr = json_to_graph(gapless_extender_graph); + auto& vg_graph = *vg_graph_ptr; return gbwtgraph::GBWTGraph(gbwt_index, vg_graph, nullptr); } diff --git a/src/unittest/genotypekit.cpp b/src/unittest/genotypekit.cpp index af9bc2a4d8..31d535dca5 100644 --- a/src/unittest/genotypekit.cpp +++ b/src/unittest/genotypekit.cpp @@ -10,6 +10,7 @@ #include "../traversal_finder.hpp" #include "xg.hpp" #include "../haplotype_extracter.hpp" +#include "support/json.hpp" namespace Catch { @@ -105,11 +106,9 @@ TEST_CASE("sites can be found with Cactus", "[genotype]") { )"; // Make an actual graph - VG graph; - Graph chunk; - json2pb(chunk, graph_json.c_str(), graph_json.size()); - graph.merge(chunk); - + auto graph_ptr = json_to_graph(graph_json); + VG& graph = *dynamic_cast(graph_ptr.get()); + // Make a CactusSnarlFinder unique_ptr finder(new CactusSnarlFinder(graph)); @@ -239,32 +238,30 @@ TEST_CASE("sites can be found with the IntegratedSnarlFinder", "[genotype][integ )"; // Make an actual graph - VG graph; - Graph chunk; - json2pb(chunk, graph_json.c_str(), graph_json.size()); - graph.merge(chunk); - + auto graph_ptr = json_to_graph(graph_json); + VG& graph = *dynamic_cast(graph_ptr.get()); + // Make an IntegratedSnarlFinder unique_ptr finder(new IntegratedSnarlFinder(graph)); - + SECTION("IntegratedSnarlFinder should find two top-level sites") { - + SnarlManager manager = finder->find_snarls(); - + auto sites = manager.top_level_snarls(); - + REQUIRE(sites.size() == 2); - + // Order them const Snarl* site_1 = sites[0]->start().node_id() > sites[1]->start().node_id() ? sites[1] : sites[0]; const Snarl* site_2 = sites[0]->start().node_id() > sites[1]->start().node_id() ? sites[0] : sites[1]; - + SECTION("the first site should be 1 fwd to 6 fwd") { REQUIRE(site_1->start().node_id() == 1); REQUIRE(site_1->start().backward() == false); REQUIRE(site_1->end().node_id() == 6); REQUIRE(site_1->end().backward() == false); - + SECTION("and should contain exactly nodes 1 through 6") { auto nodes = manager.deep_contents(site_1, graph, true).first; set correct{graph.get_node(1), graph.get_node(2), @@ -592,11 +589,9 @@ TEST_CASE("CactusSnarlFinder safely handles a single node graph", "[genotype][ca )"; // Make an actual graph - VG graph; - Graph chunk; - json2pb(chunk, graph_json.c_str(), graph_json.size()); - graph.merge(chunk); - + auto graph_ptr = json_to_graph(graph_json); + VG& graph = *dynamic_cast(graph_ptr.get()); + // Make a CactusSnarlFinder unique_ptr finder(new CactusSnarlFinder(graph)); @@ -1119,11 +1114,9 @@ TEST_CASE("CactusSnarlFinder throws an error instead of crashing when the graph )"; // Make an actual graph - VG graph; - Graph chunk; - json2pb(chunk, graph_json.c_str(), graph_json.size()); - graph.merge(chunk); - + auto graph_ptr = json_to_graph(graph_json); + VG& graph = *dynamic_cast(graph_ptr.get()); + // Make a CactusSnarlFinder unique_ptr finder(new CactusSnarlFinder(graph)); @@ -1223,11 +1216,9 @@ TEST_CASE("TrivialTraversalFinder can find traversals", "[genotype]") { )"; // Make an actual graph - VG graph; - Graph chunk; - json2pb(chunk, graph_json.c_str(), graph_json.size()); - graph.merge(chunk); - + auto graph_ptr = json_to_graph(graph_json); + auto& graph = *graph_ptr; + // Make a site Snarl site; site.mutable_start()->set_node_id(2); @@ -1235,7 +1226,7 @@ TEST_CASE("TrivialTraversalFinder can find traversals", "[genotype]") { site.set_type(ULTRABUBBLE); site.set_start_end_reachable(true); site.set_directed_acyclic_net_graph(true); - + // Make the TraversalFinder TraversalFinder* finder = new TrivialTraversalFinder(graph); diff --git a/src/unittest/genotyper.cpp b/src/unittest/genotyper.cpp index e2e9f7a142..19fa7eafd8 100644 --- a/src/unittest/genotyper.cpp +++ b/src/unittest/genotyper.cpp @@ -7,6 +7,7 @@ #include "../snarls.hpp" #include "../cactus_snarl_finder.hpp" #include "../traversal_finder.hpp" +#include "support/json.hpp" namespace vg { namespace unittest { @@ -56,11 +57,9 @@ TEST_CASE("traversals can be found from reads", "[genotyper]") { )"; // Make an actual graph - VG graph; - Graph chunk; - json2pb(chunk, graph_json.c_str(), graph_json.size()); - graph.merge(chunk); - + auto graph_ptr = json_to_graph(graph_json); + VG& graph = *dynamic_cast(graph_ptr.get()); + // Find the snarls SnarlManager manager = CactusSnarlFinder(graph).find_snarls(); diff --git a/src/unittest/haplotypes.cpp b/src/unittest/haplotypes.cpp index e441bbe197..fdbb80756f 100644 --- a/src/unittest/haplotypes.cpp +++ b/src/unittest/haplotypes.cpp @@ -6,9 +6,11 @@ #include "haplotypes.hpp" #include "xg.hpp" #include "vg.hpp" +#include "support/json.hpp" #include +namespace vg { namespace unittest { using namespace std; @@ -93,18 +95,14 @@ TEST_CASE("We can represent appropriate graphs according to linear reference", " thread_t del_ref_thread = {tm[1], tm[2], tm[4]}; thread_t del_thread = {tm[1], tm[4]}; - vg::Graph SNP_proto_graph; - json2pb(SNP_proto_graph, SNP_graph_json.c_str(), SNP_graph_json.size()); // Build the xg index xg::XG SNP_xg_index; - SNP_xg_index.from_path_handle_graph(vg::VG(SNP_proto_graph)); + SNP_xg_index.from_path_handle_graph(*json_to_graph(SNP_graph_json)); vg::path_handle_t SNP_ref_path_handle = SNP_xg_index.get_path_handle("reference"); - vg::Graph del_proto_graph; - json2pb(del_proto_graph, del_graph_json.c_str(), del_graph_json.size()); // Build the xg index xg::XG del_xg_index; - del_xg_index.from_path_handle_graph(vg::VG(del_proto_graph)); + del_xg_index.from_path_handle_graph(*json_to_graph(del_graph_json)); vg::path_handle_t del_ref_path_handle = del_xg_index.get_path_handle("reference"); // NEGATIVE SNVs @@ -159,18 +157,14 @@ TEST_CASE("We can represent appropriate graphs according to linear reference", " thread_t double_thread = {tm[1], tm[2], tm[4]}; - vg::Graph long_proto_graph; - json2pb(long_proto_graph, long_graph_json.c_str(), long_graph_json.size()); // Build the xg index xg::XG long_xg_index; - long_xg_index.from_path_handle_graph(vg::VG(long_proto_graph)); + long_xg_index.from_path_handle_graph(*json_to_graph(long_graph_json)); vg::path_handle_t long_ref_path_handle = long_xg_index.get_path_handle("reference"); - vg::Graph double_proto_graph; - json2pb(double_proto_graph, double_graph_json.c_str(), double_graph_json.size()); // Build the xg index xg::XG double_xg_index; - double_xg_index.from_path_handle_graph(vg::VG(double_proto_graph)); + double_xg_index.from_path_handle_graph(*json_to_graph(double_graph_json)); vg::path_handle_t double_ref_path_handle = double_xg_index.get_path_handle("reference"); string matching_test_file = "matching_test.slls"; @@ -383,12 +377,9 @@ TEST_CASE("We can recognize a required crossover", "[hapo-score][gbwt]") { // This graph is the start of xy2 from test/small string graph_json = R"({"node": [{"id": 1, "sequence": "CAAATAAGGCTT"}, {"id": 2, "sequence": "G"}, {"id": 3, "sequence": "GGAAATTTTC"}, {"id": 4, "sequence": "C"}, {"id": 5, "sequence": "TGGAGTTCTATTATATTCC"}, {"id": 6, "sequence": "G"}, {"id": 7, "sequence": "A"}, {"id": 8, "sequence": "ACTCTCTGGTTCCTG"}, {"id": 9, "sequence": "A"}, {"id": 10, "sequence": "G"}, {"id": 11, "sequence": "TGCTATGTGTAACTAGTAATGGTAATGGATATGTTGGGCTTTTTTCTTTGATTTATTTGAAGTGACGTTTGACAATCTATCACTAGGGGTAATGTGGGGAAATGGAAAGAATACAAGATTTGGAGCCA"}], "edge": [{"from": 1, "to": 2}, {"from": 1, "to": 3}, {"from": 2, "to": 3}, {"from": 3, "to": 4}, {"from": 3, "to": 5}, {"from": 4, "to": 5}, {"from": 5, "to": 6}, {"from": 5, "to": 7}, {"from": 6, "to": 8}, {"from": 7, "to": 8}, {"from": 8, "to": 9}, {"from": 8, "to": 10}, {"from": 9, "to": 11}, {"from": 10, "to": 11}]})"; - // Load the JSON - vg::Graph proto_graph; - json2pb(proto_graph, graph_json.c_str(), graph_json.size()); - // Build the xg index + // Load the JSON and build the xg index xg::XG xg_index; - xg_index.from_path_handle_graph(vg::VG(proto_graph)); + xg_index.from_path_handle_graph(*json_to_graph(graph_json)); gbwt::Verbosity::set(gbwt::Verbosity::SILENT); gbwt::DynamicGBWT* gbwt_index = new gbwt::DynamicGBWT; @@ -480,3 +471,4 @@ TEST_CASE("We can recognize a required crossover", "[hapo-score][gbwt]") { } } +} diff --git a/src/unittest/indexed_vg.cpp b/src/unittest/indexed_vg.cpp index 7f74d92193..1f1a687abc 100644 --- a/src/unittest/indexed_vg.cpp +++ b/src/unittest/indexed_vg.cpp @@ -10,6 +10,7 @@ #include "../utility.hpp" #include "../algorithms/id_sort.hpp" #include "support/random_graph.hpp" +#include "support/json.hpp" #include "catch.hpp" namespace vg { diff --git a/src/unittest/mapper.cpp b/src/unittest/mapper.cpp index 2caf42d076..dcd830a7ed 100644 --- a/src/unittest/mapper.cpp +++ b/src/unittest/mapper.cpp @@ -10,6 +10,7 @@ #include "xg.hpp" #include "../build_index.hpp" #include "catch.hpp" +#include "support/json.hpp" #include "../algorithms/alignment_path_offsets.hpp" namespace vg { @@ -26,13 +27,8 @@ TEST_CASE( "Mapper can map to a one-node graph", "[mapping][mapper]" ) { ] })"; - // Load the JSON - Graph proto_graph; - json2pb(proto_graph, graph_json.c_str(), graph_json.size()); - - // Make it into a VG - VG graph; - graph.extend(proto_graph); + // Load the JSON and make it into a VG + auto graph = json_to_graph(graph_json); // Make GCSA quiet gcsa::Verbosity::set(gcsa::Verbosity::SILENT); @@ -246,13 +242,8 @@ TEST_CASE( "Mapper finds optimal mapping for read starting with node-border MEM" {"position":{"node_id":1445},"rank":1060}]}]} )"; - // Load the JSON - Graph proto_graph; - json2pb(proto_graph, graph_json.c_str(), graph_json.size()); - - // Make it into a VG - VG graph; - graph.extend(proto_graph); + // Load the JSON and make it into a VG + auto graph = json_to_graph(graph_json); // Make GCSA quiet gcsa::Verbosity::set(gcsa::Verbosity::SILENT); @@ -312,13 +303,8 @@ TEST_CASE( "Mapper can annotate positions correctly on both strands", "[mapper][ ]} )"; - // Load the JSON - Graph proto_graph; - json2pb(proto_graph, graph_json.c_str(), graph_json.size()); - - // Make it into a VG - VG graph; - graph.extend(proto_graph); + // Load the JSON and make it into a VG + auto graph = json_to_graph(graph_json); // Make GCSA quiet gcsa::Verbosity::set(gcsa::Verbosity::SILENT); @@ -328,11 +314,11 @@ TEST_CASE( "Mapper can annotate positions correctly on both strands", "[mapper][ gcsa::LCPArray* lcpidx = nullptr; // Build the GCSA index - build_gcsa_lcp(graph, gcsaidx, lcpidx, 16, 3); + build_gcsa_lcp(*graph, gcsaidx, lcpidx, 16, 3); // Build the xg index xg::XG xg_index; - xg_index.from_path_handle_graph(graph); + xg_index.from_path_handle_graph(*graph); // Make a multipath mapper to map against the graph. Mapper mapper(&xg_index, gcsaidx, lcpidx); diff --git a/src/unittest/minimizer_mapper.cpp b/src/unittest/minimizer_mapper.cpp index e2e25db870..aab6a8fedc 100644 --- a/src/unittest/minimizer_mapper.cpp +++ b/src/unittest/minimizer_mapper.cpp @@ -7,6 +7,7 @@ #include "../io/json2graph.hpp" #include #include "../minimizer_mapper.hpp" +#include "support/json.hpp" #include "../build_index.hpp" #include "../integrated_snarl_finder.hpp" #include "../gbwt_extender.hpp" @@ -452,9 +453,7 @@ TEST_CASE("MinimizerMapper can map an empty string between odd points", "[giraff })"; // TODO: Write a json_to_handle_graph - vg::Graph proto_graph; - json2pb(proto_graph, graph_json.c_str(), graph_json.size()); - auto graph = vg::VG(proto_graph); + auto graph = json_to_graph(graph_json); Alignment aln; aln.set_sequence(""); @@ -462,7 +461,7 @@ TEST_CASE("MinimizerMapper can map an empty string between odd points", "[giraff pos_t left_anchor {55511921, false, 5}; // This is on the final base of the node pos_t right_anchor {55511925, false, 6}; - TestMinimizerMapper::align_sequence_between(left_anchor, right_anchor, 100, 20, &graph, &aligner, aln); + TestMinimizerMapper::align_sequence_between(left_anchor, right_anchor, 100, 20, graph.get(), &aligner, aln); // Make sure we get the right alignment. We should see the last base of '21 and go '21 to '24 to '25 and delete everything REQUIRE(aln.path().mapping_size() == 3); @@ -494,9 +493,7 @@ TEST_CASE("MinimizerMapper can map with an initial deletion", "[giraffe][mapping })"; // TODO: Write a json_to_handle_graph - vg::Graph proto_graph; - json2pb(proto_graph, graph_json.c_str(), graph_json.size()); - auto graph = vg::VG(proto_graph); + auto graph = json_to_graph(graph_json); Alignment aln; aln.set_sequence("CATTAG"); @@ -541,9 +538,7 @@ TEST_CASE("MinimizerMapper can map with an initial deletion on a multi-base node })"; // TODO: Write a json_to_handle_graph - vg::Graph proto_graph; - json2pb(proto_graph, graph_json.c_str(), graph_json.size()); - auto graph = vg::VG(proto_graph); + auto graph = json_to_graph(graph_json); Alignment aln; aln.set_sequence("CATTAG"); @@ -588,9 +583,7 @@ TEST_CASE("MinimizerMapper can map right off the past-the-end base", "[giraffe][ })"; // TODO: Write a json_to_handle_graph - vg::Graph proto_graph; - json2pb(proto_graph, graph_json.c_str(), graph_json.size()); - auto graph = vg::VG(proto_graph); + auto graph = json_to_graph(graph_json); Alignment aln; aln.set_sequence("CATTAG"); @@ -641,9 +634,7 @@ TEST_CASE("MinimizerMapper can find a significant indel instead of a tempting so })"; // TODO: Write a json_to_handle_graph - vg::Graph proto_graph; - json2pb(proto_graph, graph_json.c_str(), graph_json.size()); - auto graph = vg::VG(proto_graph); + auto graph = json_to_graph(graph_json); Alignment aln; aln.set_sequence("TTGAAAACCTGATATGTCTTATTTTTCTAACTATGGAATTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTGAGACGGAGTCTCGCTCTGTCGCCCAGGCTGGAGTGCAGTGGCGCGATCTCGGCTCACTGCAAGCTCCGCCTCCCGGGTTCACGCCATTCTCCTGCCTCAGCCTCCCGAGTAGCTGGGACTACAGGCGCCCGCTACCACGCCCGGCTAATTTTTTGTATTTTTTTT"); diff --git a/src/unittest/multipath_alignment_graph.cpp b/src/unittest/multipath_alignment_graph.cpp index bea5f687aa..815d835638 100644 --- a/src/unittest/multipath_alignment_graph.cpp +++ b/src/unittest/multipath_alignment_graph.cpp @@ -11,6 +11,7 @@ #include "../snarl_distance_index.hpp" #include "catch.hpp" #include "support/test_aligner.hpp" +#include "support/json.hpp" @@ -47,12 +48,8 @@ TEST_CASE( "MultipathAlignmentGraph::align handles tails correctly", "[multipath })"; // Load the JSON - Graph proto_graph; - json2pb(proto_graph, graph_json.c_str(), graph_json.size()); - // Make it into a VG - VG vg; - vg.extend(proto_graph); + auto vg = json_to_graph(graph_json); // Make snarls on it CactusSnarlFinder bubble_finder(vg); diff --git a/src/unittest/multipath_mapper.cpp b/src/unittest/multipath_mapper.cpp index be6d3b6194..1113a22202 100644 --- a/src/unittest/multipath_mapper.cpp +++ b/src/unittest/multipath_mapper.cpp @@ -10,6 +10,7 @@ #include "xg.hpp" #include "vg.hpp" #include "catch.hpp" +#include "support/json.hpp" namespace vg { namespace unittest { @@ -122,12 +123,8 @@ TEST_CASE( "MultipathMapper::query_cluster_graphs works", "[multipath][mapping][ })"; // Load the JSON - Graph proto_graph; - json2pb(proto_graph, graph_json.c_str(), graph_json.size()); - // Make it into a VG - VG graph; - graph.extend(proto_graph); + auto graph = json_to_graph(graph_json); // Make GCSA quiet gcsa::Verbosity::set(gcsa::Verbosity::SILENT); @@ -275,12 +272,8 @@ TEST_CASE( "MultipathMapper can map to a one-node graph", "[multipath][mapping][ })"; // Load the JSON - Graph proto_graph; - json2pb(proto_graph, graph_json.c_str(), graph_json.size()); - // Make it into a VG - VG graph; - graph.extend(proto_graph); + auto graph = json_to_graph(graph_json); // Make GCSA quiet gcsa::Verbosity::set(gcsa::Verbosity::SILENT); @@ -290,11 +283,11 @@ TEST_CASE( "MultipathMapper can map to a one-node graph", "[multipath][mapping][ gcsa::LCPArray* lcpidx = nullptr; // Build the GCSA index - build_gcsa_lcp(graph, gcsaidx, lcpidx, 16, 3); + build_gcsa_lcp(*graph, gcsaidx, lcpidx, 16, 3); // Build the xg index xg::XG xg_index; - xg_index.from_path_handle_graph(graph); + xg_index.from_path_handle_graph(*graph); // Make a multipath mapper to map against the graph. MultipathMapper mapper(&xg_index, gcsaidx, lcpidx); @@ -426,12 +419,8 @@ TEST_CASE( "MultipathMapper can work on a bigger graph", "[multipath][mapping][m string graph_json = R"({"node":[{"sequence":"CTTCTCATCCCTCCTCAAGGGCCTTTAACTACTCCACATCCAAAGCTACCCAGGCCATTTTAAGTTTCCTGTGGACTAAGGACAAAGGTGCGGGGAGATG","id":12},{"sequence":"A","id":2},{"sequence":"CAAATAAGGCTTGGAAATTTTCTGGAGTTCTATTATATTCCAACTCTCTGGTTCCTGGTGCTATGTGTAACTAGTAATGGTAATGGATATGTTGGGCTTT","id":3},{"sequence":"TTTCTTTGATTTATTTGAAGTGACGTTTGACAATCTATCACTAGGGGTAATGTGGGGAAATGGAAAGAATACAAGATTTGGAGCCAGACAAATCTGGGTT","id":4},{"sequence":"CAAATCCTCACTTTGCCACATATTAGCCATGTGACTTTGAACAAGTTAGTTAATCTCTCTGAACTTCAGTTTAATTATCTCTAATATGGAGATGATACTA","id":5},{"sequence":"CTGACAGCAGAGGTTTGCTGTGAAGATTAAATTAGGTGATGCTTGTAAAGCTCAGGGAATAGTGCCTGGCATAGAGGAAAGCCTCTGACAACTGGTAGTT","id":6},{"sequence":"ACTGTTATTTACTATGAATCCTCACCTTCCTTGACTTCTTGAAACATTTGGCTATTGACCTCTTTCCTCCTTGAGGCTCTTCTGGCTTTTCATTGTCAAC","id":7},{"sequence":"ACAGTCAACGCTCAATACAAGGGACATTAGGATTGGCAGTAGCTCAGAGATCTCTCTGCTCACCGTGATCTTCAAGTTTGAAAATTGCATCTCAAATCTA","id":8},{"sequence":"AGACCCAGAGGGCTCACCCAGAGTCGAGGCTCAAGGACAGCTCTCCTTTGTGTCCAGAGTGTATACGATGTAACTCTGTTCGGGCACTGGTGAAAGATAA","id":9},{"sequence":"CAGAGGAAATGCCTGGCTTTTTATCAGAACATGTTTCCAAGCTTATCCCTTTTCCCAGCTCTCCTTGTCCCTCCCAAGATCTCTTCACTGGCCTCTTATC","id":10},{"sequence":"TTTACTGTTACCAAATCTTTCCAGAAGCTGCTCTTTCCCTCAATTGTTCATTTGTCTTCTTGTCCAGGAATGAACCACTGCTCTCTTCTTGTCAGATCAG","id":11}],"path":[{"name":"x","mapping":[{"position":{"node_id":3},"edit":[{"from_length":100,"to_length":100}],"rank":1},{"position":{"node_id":4},"edit":[{"from_length":100,"to_length":100}],"rank":2},{"position":{"node_id":5},"edit":[{"from_length":100,"to_length":100}],"rank":3},{"position":{"node_id":6},"edit":[{"from_length":100,"to_length":100}],"rank":4},{"position":{"node_id":7},"edit":[{"from_length":100,"to_length":100}],"rank":5},{"position":{"node_id":8},"edit":[{"from_length":100,"to_length":100}],"rank":6},{"position":{"node_id":9},"edit":[{"from_length":100,"to_length":100}],"rank":7},{"position":{"node_id":10},"edit":[{"from_length":100,"to_length":100}],"rank":8},{"position":{"node_id":11},"edit":[{"from_length":100,"to_length":100}],"rank":9},{"position":{"node_id":12},"edit":[{"from_length":100,"to_length":100}],"rank":10},{"position":{"node_id":2},"edit":[{"from_length":1,"to_length":1}],"rank":11}]}],"edge":[{"from":12,"to":2},{"from":3,"to":4},{"from":4,"to":5},{"from":5,"to":6},{"from":6,"to":7},{"from":7,"to":8},{"from":8,"to":9},{"from":9,"to":10},{"from":10,"to":11},{"from":11,"to":12}]})"; // Load the JSON - Graph proto_graph; - json2pb(proto_graph, graph_json.c_str(), graph_json.size()); - // Make it into a VG - VG graph; - graph.extend(proto_graph); + auto graph = json_to_graph(graph_json); // Make GCSA quiet gcsa::Verbosity::set(gcsa::Verbosity::SILENT); @@ -441,7 +430,7 @@ TEST_CASE( "MultipathMapper can work on a bigger graph", "[multipath][mapping][m gcsa::LCPArray* lcpidx = nullptr; // Build the GCSA index - build_gcsa_lcp(graph, gcsaidx, lcpidx, 16, 3); + build_gcsa_lcp(*graph, gcsaidx, lcpidx, 16, 3); // Build the xg index xg::XG xg_index; diff --git a/src/unittest/path_component_index.cpp b/src/unittest/path_component_index.cpp index 058f4bf9c1..bfdda70d64 100644 --- a/src/unittest/path_component_index.cpp +++ b/src/unittest/path_component_index.cpp @@ -9,6 +9,7 @@ #include "xg.hpp" #include "vg.hpp" #include "vg/io/json2pb.h" +#include "support/json.hpp" #include namespace vg { @@ -18,13 +19,9 @@ namespace unittest { string graph_json = R"({"node": [{"sequence": "AAACCC", "id": 1}, {"sequence": "CACACA", "id": 2}, {"sequence": "CACACA", "id": 3}, {"sequence": "TTTTGG", "id": 4}, {"sequence": "ACGTAC", "id": 5}], "path": [{"name": "one", "mapping": [{"position": {"node_id": 1}, "rank": 1}, {"position": {"node_id": 2}, "rank": 2}]}, {"name": "three", "mapping": [{"position": {"node_id": 2}, "rank": 1}, {"position": {"node_id": 3}, "rank": 2}]}, {"name": "two", "mapping": [{"position": {"node_id": 4}, "rank": 1}, {"position": {"node_id": 5}, "rank": 2}]}], "edge": [{"from": 1, "to": 2}, {"from": 2, "to": 3}, {"from": 4, "to": 5}]})"; - // Load the JSON - Graph proto_graph; - json2pb(proto_graph, graph_json.c_str(), graph_json.size()); - - // Build the xg index + // Load the JSON and build the xg index xg::XG xg_index; - xg_index.from_path_handle_graph(VG(proto_graph)); + xg_index.from_path_handle_graph(*json_to_graph(graph_json)); unordered_set comp_1; diff --git a/src/unittest/phase_unfolder.cpp b/src/unittest/phase_unfolder.cpp index 0c79972941..67c079ed19 100644 --- a/src/unittest/phase_unfolder.cpp +++ b/src/unittest/phase_unfolder.cpp @@ -14,6 +14,7 @@ #include "../phase_unfolder.hpp" #include "vg/io/json2pb.h" #include "xg.hpp" +#include "support/json.hpp" #include "catch.hpp" @@ -210,10 +211,8 @@ const std::string unfolder_graph_path = R"( TEST_CASE("PhaseUnfolder can unfold XG paths", "[phaseunfolder][indexing]") { // Build an XG index with a path. - Graph graph_with_path; - json2pb(graph_with_path, unfolder_graph_path.c_str(), unfolder_graph_path.size()); xg::XG xg_index; - xg_index.from_path_handle_graph(VG(graph_with_path)); + xg_index.from_path_handle_graph(*json_to_graph(unfolder_graph_path)); // Build an empty GBWT index. gbwt::GBWT gbwt_index; @@ -223,10 +222,7 @@ TEST_CASE("PhaseUnfolder can unfold XG paths", "[phaseunfolder][indexing]") { PhaseUnfolder unfolder(xg_index, gbwt_index, next_id); // Build a VG graph. - VG vg_graph; - Graph temp_graph; - json2pb(temp_graph, unfolder_graph.c_str(), unfolder_graph.size()); - vg_graph.merge(temp_graph); + auto vg_graph = json_to_graph(unfolder_graph); // Remove branching regions from the VG graph, including the last node, // but keep the edge (1, 6) in the graph. @@ -255,10 +251,8 @@ TEST_CASE("PhaseUnfolder can unfold XG paths", "[phaseunfolder][indexing]") { TEST_CASE("PhaseUnfolder can restore XG paths", "[phaseunfolder][indexing]") { // Build an XG index with a path. - Graph graph_with_path; - json2pb(graph_with_path, unfolder_graph_path.c_str(), unfolder_graph_path.size()); xg::XG xg_index; - xg_index.from_path_handle_graph(VG(graph_with_path)); + xg_index.from_path_handle_graph(*json_to_graph(unfolder_graph_path)); // Build an empty GBWT index. gbwt::GBWT gbwt_index; @@ -268,10 +262,7 @@ TEST_CASE("PhaseUnfolder can restore XG paths", "[phaseunfolder][indexing]") { PhaseUnfolder unfolder(xg_index, gbwt_index, next_id); // Build a VG graph. - VG vg_graph; - Graph temp_graph; - json2pb(temp_graph, unfolder_graph.c_str(), unfolder_graph.size()); - vg_graph.merge(temp_graph); + auto vg_graph = json_to_graph(unfolder_graph); // Remove branching regions from the VG graph, including the last node, // but keep the edge (1, 6) in the graph. @@ -334,10 +325,7 @@ TEST_CASE("PhaseUnfolder can unfold GBWT threads", "[phaseunfolder][indexing]") PhaseUnfolder unfolder(xg_index, gbwt_index, next_id); // Build a VG graph. - VG vg_graph; - Graph temp_graph; - json2pb(temp_graph, unfolder_graph.c_str(), unfolder_graph.size()); - vg_graph.merge(temp_graph); + auto vg_graph = json_to_graph(unfolder_graph); // Remove branching regions from the VG graph, including the last node, // but keep the edge (1, 6) in the graph. @@ -366,10 +354,8 @@ TEST_CASE("PhaseUnfolder can unfold GBWT threads", "[phaseunfolder][indexing]") TEST_CASE("PhaseUnfolder can unfold both XG paths and GBWT threads", "[phaseunfolder][indexing]") { // Build an XG index with a path. - Graph graph_with_path; - json2pb(graph_with_path, unfolder_graph_path.c_str(), unfolder_graph_path.size()); xg::XG xg_index; - xg_index.from_path_handle_graph(VG(graph_with_path)); + xg_index.from_path_handle_graph(*json_to_graph(unfolder_graph_path)); // Build a GBWT with three threads including a duplicate. We want to have // only one instance of short_path unfolded, but we want separate copies @@ -401,10 +387,7 @@ TEST_CASE("PhaseUnfolder can unfold both XG paths and GBWT threads", "[phaseunfo PhaseUnfolder unfolder(xg_index, gbwt_index, next_id); // Build a VG graph. - VG vg_graph; - Graph temp_graph; - json2pb(temp_graph, unfolder_graph.c_str(), unfolder_graph.size()); - vg_graph.merge(temp_graph); + auto vg_graph = json_to_graph(unfolder_graph); // Remove branching regions from the VG graph, including the last node, // but keep the edge (1, 6) in the graph. diff --git a/src/unittest/readfilter.cpp b/src/unittest/readfilter.cpp index cc1562f3f3..49ef3bb389 100644 --- a/src/unittest/readfilter.cpp +++ b/src/unittest/readfilter.cpp @@ -5,6 +5,7 @@ #include "catch.hpp" #include "readfilter.hpp" #include "xg.hpp" +#include "support/json.hpp" namespace vg { namespace unittest { @@ -45,12 +46,9 @@ TEST_CASE("reads with ambiguous ends can be trimmed", "[filter]") { )"; // Load it into Protobuf - Graph chunk; - json2pb(chunk, graph_json.c_str(), graph_json.size()); - // Pass it over to XG xg::XG index; - index.from_path_handle_graph(VG(chunk)); + index.from_path_handle_graph(*json_to_graph(graph_json)); // Make a ReadFilter; ReadFilter filter; diff --git a/src/unittest/snarl_distance_index.cpp b/src/unittest/snarl_distance_index.cpp index 093c45d0aa..947117dbd4 100644 --- a/src/unittest/snarl_distance_index.cpp +++ b/src/unittest/snarl_distance_index.cpp @@ -14,6 +14,7 @@ #include "catch.hpp" #include "support/random_graph.hpp" #include "support/randomness.hpp" +#include "support/json.hpp" #include "../snarl_distance_index.hpp" #include "../integrated_snarl_finder.hpp" #include "../genotypekit.hpp" @@ -3754,9 +3755,7 @@ namespace vg { // Make an actual graph VG graph; - Graph chunk; - json2pb(chunk, graph_json.c_str(), graph_json.size()); - graph.extend(chunk); + auto chunk_ptr = json_to_graph(graph_json); // We need to see the path. REQUIRE(graph.paths.size() == 1); @@ -4014,9 +4013,7 @@ namespace vg { // Make an actual graph VG graph; - Graph chunk; - json2pb(chunk, graph_json.c_str(), graph_json.size()); - graph.extend(chunk); + auto chunk_ptr = json_to_graph(graph_json); IntegratedSnarlFinder snarl_finder(graph); SnarlDistanceIndex distance_index; @@ -4127,9 +4124,7 @@ namespace vg { // Make an actual graph VG graph; - Graph chunk; - json2pb(chunk, graph_json.c_str(), graph_json.size()); - graph.extend(chunk); + auto chunk_ptr = json_to_graph(graph_json); IntegratedSnarlFinder snarl_finder(graph); SnarlDistanceIndex distance_index; @@ -4276,9 +4271,7 @@ namespace vg { // Make an actual graph VG graph; - Graph chunk; - json2pb(chunk, graph_json.c_str(), graph_json.size()); - graph.extend(chunk); + auto chunk_ptr = json_to_graph(graph_json); IntegratedSnarlFinder snarl_finder(graph); SnarlDistanceIndex distance_index; @@ -4405,9 +4398,7 @@ namespace vg { // Make an actual graph VG graph; - Graph chunk; - json2pb(chunk, graph_json.c_str(), graph_json.size()); - graph.extend(chunk); + auto chunk_ptr = json_to_graph(graph_json); IntegratedSnarlFinder snarl_finder(graph); SnarlDistanceIndex distance_index; @@ -4514,9 +4505,7 @@ namespace vg { // Make an actual graph VG graph; - Graph chunk; - json2pb(chunk, graph_json.c_str(), graph_json.size()); - graph.extend(chunk); + auto chunk_ptr = json_to_graph(graph_json); IntegratedSnarlFinder snarl_finder(graph); SnarlDistanceIndex distance_index; @@ -4618,9 +4607,7 @@ namespace vg { // Make an actual graph VG graph; - Graph chunk; - json2pb(chunk, graph_json.c_str(), graph_json.size()); - graph.extend(chunk); + auto chunk_ptr = json_to_graph(graph_json); IntegratedSnarlFinder snarl_finder(graph); SnarlDistanceIndex distance_index; @@ -4788,9 +4775,7 @@ namespace vg { // Make an actual graph VG graph; - Graph chunk; - json2pb(chunk, graph_json.c_str(), graph_json.size()); - graph.extend(chunk); + auto chunk_ptr = json_to_graph(graph_json); IntegratedSnarlFinder snarl_finder(graph); SnarlDistanceIndex distance_index; @@ -4911,9 +4896,7 @@ namespace vg { // Make an actual graph VG graph; - Graph chunk; - json2pb(chunk, graph_json.c_str(), graph_json.size()); - graph.extend(chunk); + auto chunk_ptr = json_to_graph(graph_json); IntegratedSnarlFinder snarl_finder(graph); SnarlDistanceIndex distance_index; diff --git a/src/unittest/snarls.cpp b/src/unittest/snarls.cpp index c2f5030326..4d6be27383 100644 --- a/src/unittest/snarls.cpp +++ b/src/unittest/snarls.cpp @@ -13,6 +13,7 @@ #include "catch.hpp" #include "support/random_graph.hpp" #include "support/randomness.hpp" +#include "support/json.hpp" #include "../snarls.hpp" #include "../cactus_snarl_finder.hpp" #include "../integrated_snarl_finder.hpp" @@ -1701,9 +1702,7 @@ namespace vg { VG graph; // Load up the graph - Graph g; - json2pb(g, graph_json.c_str(), graph_json.size()); - graph.extend(g); + auto g_ptr = json_to_graph(graph_json); // Define the one snarl Snarl snarl1; @@ -1834,9 +1833,7 @@ namespace vg { VG graph; // Load up the graph - Graph g; - json2pb(g, graph_json.c_str(), graph_json.size()); - graph.extend(g); + auto g_ptr = json_to_graph(graph_json); // Load the snarls Snarl snarl1, snarl2, snarl3; @@ -1920,9 +1917,7 @@ namespace vg { // Make an actual graph VG graph; - Graph chunk; - json2pb(chunk, graph_json.c_str(), graph_json.size()); - graph.extend(chunk); + auto chunk_ptr = json_to_graph(graph_json); // We need to see the path. REQUIRE(graph.paths.size() == 1); @@ -2045,9 +2040,7 @@ namespace vg { // Make an actual graph VG graph; - Graph chunk; - json2pb(chunk, graph_json.c_str(), graph_json.size()); - graph.extend(chunk); + auto chunk_ptr = json_to_graph(graph_json); SnarlManager snarl_manager = CactusSnarlFinder(graph).find_snarls(); @@ -2132,9 +2125,7 @@ namespace vg { // Make an actual graph VG graph; - Graph chunk; - json2pb(chunk, graph_json.c_str(), graph_json.size()); - graph.extend(chunk); + auto chunk_ptr = json_to_graph(graph_json); SnarlManager snarl_manager = CactusSnarlFinder(graph).find_snarls(); @@ -2251,9 +2242,7 @@ namespace vg { // Make an actual graph VG graph; - Graph chunk; - json2pb(chunk, graph_json.c_str(), graph_json.size()); - graph.extend(chunk); + auto chunk_ptr = json_to_graph(graph_json); SnarlManager snarl_manager = CactusSnarlFinder(graph).find_snarls(); @@ -2362,9 +2351,7 @@ namespace vg { // Make an actual graph VG graph; - Graph chunk; - json2pb(chunk, graph_json.c_str(), graph_json.size()); - graph.extend(chunk); + auto chunk_ptr = json_to_graph(graph_json); SnarlManager snarl_manager = CactusSnarlFinder(graph).find_snarls(); @@ -2420,9 +2407,7 @@ namespace vg { // Make an actual graph VG graph; - Graph chunk; - json2pb(chunk, graph_json.c_str(), graph_json.size()); - graph.extend(chunk); + auto chunk_ptr = json_to_graph(graph_json); SnarlManager snarl_manager = CactusSnarlFinder(graph).find_snarls(); @@ -2498,9 +2483,7 @@ namespace vg { // Make an actual graph VG graph; - Graph chunk; - json2pb(chunk, graph_json.c_str(), graph_json.size()); - graph.extend(chunk); + auto chunk_ptr = json_to_graph(graph_json); SnarlManager snarl_manager = CactusSnarlFinder(graph).find_snarls(); @@ -2560,9 +2543,7 @@ namespace vg { // Make an actual graph VG graph; - Graph chunk; - json2pb(chunk, graph_json.c_str(), graph_json.size()); - graph.extend(chunk); + auto chunk_ptr = json_to_graph(graph_json); SnarlManager snarl_manager = CactusSnarlFinder(graph).find_snarls(); @@ -2772,9 +2753,7 @@ namespace vg { // Make an actual graph VG graph; - Graph chunk; - json2pb(chunk, graph_json.c_str(), graph_json.size()); - graph.extend(chunk); + auto chunk_ptr = json_to_graph(graph_json); SnarlManager snarl_manager = CactusSnarlFinder(graph).find_snarls(); #ifdef debug @@ -3924,9 +3903,7 @@ namespace vg { // Make an actual graph VG graph; - Graph chunk; - json2pb(chunk, graph_json.c_str(), graph_json.size()); - graph.extend(chunk); + auto chunk_ptr = json_to_graph(graph_json); assert(graph.is_valid()); SECTION( "PathTraversalFinder can find simple forward traversals") { diff --git a/src/unittest/variant_adder.cpp b/src/unittest/variant_adder.cpp index afe3353e4b..69d861ebbf 100644 --- a/src/unittest/variant_adder.cpp +++ b/src/unittest/variant_adder.cpp @@ -10,6 +10,7 @@ #include "../utility.hpp" #include "../path.hpp" #include "vg/io/json2pb.h" +#include "support/json.hpp" #include #include @@ -52,15 +53,11 @@ ref 5 rs1337 A G 29 PASS . GT ] })"; - // Load the JSON - Graph proto_graph; - json2pb(proto_graph, graph_json.c_str(), graph_json.size()); - - // Make it into a VG - VG graph; - graph.extend(proto_graph); - - + // Load the JSON and make it into a VG + auto graph_ptr = json_to_graph(graph_json); + VG& graph = *dynamic_cast(graph_ptr.get()); + + // Make a VariantAdder VariantAdder adder(graph); // Fail to add the variants to the graph @@ -99,14 +96,10 @@ ref 5 rs1337 A G 29 PASS . GT 0/1 ] })"; - // Load the JSON - Graph proto_graph; - json2pb(proto_graph, graph_json.c_str(), graph_json.size()); - - // Make it into a VG - VG graph; - graph.extend(proto_graph); - + // Load the JSON and make it into a VG + auto graph_ptr = json_to_graph(graph_json); + VG& graph = *dynamic_cast(graph_ptr.get()); + // Make a VariantAdder VariantAdder adder(graph); // Add the variants to the graph @@ -153,14 +146,10 @@ ref 5 rs1337 AAAAAAAAAAAAAAAAAAAAA A 29 PASS . GT 0/1 ] })"; - // Load the JSON - Graph proto_graph; - json2pb(proto_graph, graph_json.c_str(), graph_json.size()); - - // Make it into a VG - VG graph; - graph.extend(proto_graph); - + // Load the JSON and make it into a VG + auto graph_ptr = json_to_graph(graph_json); + VG& graph = *dynamic_cast(graph_ptr.get()); + // Make a VariantAdder VariantAdder adder(graph); // Add the variants to the graph @@ -214,16 +203,12 @@ ref 5 rs1337 AAAAAAAAAAAAAAAAAAAAA A 29 PASS . GT 0/1 ] })"; - // Load the JSON - Graph proto_graph; - json2pb(proto_graph, graph_json.c_str(), graph_json.size()); - - // Make it into a VG - VG graph; - graph.extend(proto_graph); - + // Load the JSON and make it into a VG + auto graph_ptr = json_to_graph(graph_json); + VG& graph = *dynamic_cast(graph_ptr.get()); + SECTION ("should work when the graph is as given") { - + // Make a VariantAdder VariantAdder adder(graph); // Add the variants to the graph @@ -294,14 +279,10 @@ ref 5 rs1337 AAAAAAAAAAAAAAAAAAAAA AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA 29 ] })"; - // Load the JSON - Graph proto_graph; - json2pb(proto_graph, graph_json.c_str(), graph_json.size()); - - // Make it into a VG - VG graph; - graph.extend(proto_graph); - + // Load the JSON and make it into a VG + auto graph_ptr = json_to_graph(graph_json); + VG& graph = *dynamic_cast(graph_ptr.get()); + // Make a VariantAdder VariantAdder adder(graph); adder.skip_structural_duplications = true; @@ -324,14 +305,10 @@ TEST_CASE( "The smart aligner works on very large inserts", "[variantadder]" ) { "node": [{"id": 1, "sequence": "GCGCAAAAAAAAAAAAAAAAAAAAAGCGC"}] })"; - // Load the JSON - Graph proto_graph; - json2pb(proto_graph, graph_json.c_str(), graph_json.size()); - - // Make it into a VG - VG graph; - graph.extend(proto_graph); - + // Load the JSON and make it into a VG + auto graph_ptr = json_to_graph(graph_json); + VG& graph = *dynamic_cast(graph_ptr.get()); + // Make a VariantAdder VariantAdder adder(graph); vector order = handlealgs::topological_order(&adder.get_graph()); @@ -404,14 +381,10 @@ TEST_CASE( "The smart aligner should use mapping offsets on huge deletions", "[v } graph_json = regex_replace(graph_json, std::regex("<10kAs>"), a_stream.str()); - // Load the JSON - Graph proto_graph; - json2pb(proto_graph, graph_json.c_str(), graph_json.size()); - - // Make it into a VG - VG graph; - graph.extend(proto_graph); - + // Load the JSON and make it into a VG + auto graph_ptr = json_to_graph(graph_json); + VG& graph = *dynamic_cast(graph_ptr.get()); + // Make a VariantAdder VariantAdder adder(graph); vector order = handlealgs::topological_order(&adder.get_graph()); @@ -492,14 +465,10 @@ TEST_CASE( "The smart aligner should find existing huge deletions", "[variantadd } graph_json = regex_replace(graph_json, std::regex("<10kAs>"), a_stream.str()); - // Load the JSON - Graph proto_graph; - json2pb(proto_graph, graph_json.c_str(), graph_json.size()); - - // Make it into a VG - VG graph; - graph.extend(proto_graph); - + // Load the JSON and make it into a VG + auto graph_ptr = json_to_graph(graph_json); + VG& graph = *dynamic_cast(graph_ptr.get()); + // Make a VariantAdder VariantAdder adder(graph); vector order = handlealgs::topological_order(&adder.get_graph()); @@ -572,17 +541,13 @@ TEST_CASE( "The smart aligner should use deletion edits on medium deletions", "[ } graph_json = regex_replace(graph_json, std::regex("<100As>"), a_stream.str()); - // Load the JSON - Graph proto_graph; - json2pb(proto_graph, graph_json.c_str(), graph_json.size()); - - // Make it into a VG - VG graph; - graph.extend(proto_graph); - + // Load the JSON and make it into a VG + auto graph_ptr = json_to_graph(graph_json); + VG& graph = *dynamic_cast(graph_ptr.get()); + // Make a VariantAdder VariantAdder adder(graph); - + // Make a deleted version (only 21 As) string deleted = "GCGCAAAAAAAAAAAAAAAAAAAAAGCGC"; diff --git a/src/unittest/vg.cpp b/src/unittest/vg.cpp index 9beb3e1ca7..81b3b96424 100644 --- a/src/unittest/vg.cpp +++ b/src/unittest/vg.cpp @@ -9,6 +9,7 @@ #include "../algorithms/normalize.hpp" #include "../algorithms/disjoint_components.hpp" #include "handle.hpp" +#include "support/json.hpp" namespace vg { namespace unittest { @@ -17,10 +18,8 @@ using namespace std; // Turn a JSON string into a VG graph VG string_to_graph(const string& json) { - VG graph; - Graph chunk; - json2pb(chunk, json.c_str(), json.size()); - graph.merge(chunk); + auto graph_ptr = json_to_graph(json); + VG& graph = *dynamic_cast(graph_ptr.get()); return graph; } diff --git a/src/unittest/vg_algorithms.cpp b/src/unittest/vg_algorithms.cpp index b4fc736734..380946e5eb 100644 --- a/src/unittest/vg_algorithms.cpp +++ b/src/unittest/vg_algorithms.cpp @@ -28,6 +28,7 @@ #include "../xg.hpp" #include #include "vg/io/json2pb.h" +#include "support/json.hpp" using namespace google::protobuf; @@ -1092,11 +1093,7 @@ TEST_CASE( "Connecting graph extraction works on a cool loop without leaving ext {"edge": [{"from": "185927720", "to": "185927722"}, {"from": "185927721", "from_start": true, "to": "185927722"}, {"from": "185927722", "to": "186681786", "to_end": true}, {"from": "185927722", "to": "185927723"}, {"from": "186681786", "to": "186683083"}, {"from": "186681786", "from_start": true, "to": "186681787", "to_end": true}, {"from": "186681787", "to": "186683069", "to_end": true}, {"from": "186681787", "from_start": true, "to": "186681789"}, {"from": "186681787", "from_start": true, "to": "186681788", "to_end": true}, {"from": "186681788", "from_start": true, "to": "186681790", "to_end": true}, {"from": "186681789", "to": "186681790", "to_end": true}, {"from": "186681790", "from_start": true, "to": "186681792", "to_end": true}, {"from": "186683069", "from_start": true, "to": "186683079", "to_end": true}, {"from": "186683079", "from_start": true, "to": "186683080", "to_end": true}, {"from": "186683080", "from_start": true, "to": "186683081", "to_end": true}, {"from": "186683081", "from_start": true, "to": "186683083", "to_end": true}], "node": [{"id": "185927720", "sequence": "G"}, {"id": "185927721", "sequence": "A"}, {"id": "185927722", "sequence": "ACCGGG"}, {"id": "185927723", "sequence": "AGTGGGGG"}, {"id": "186681786", "sequence": "C"}, {"id": "186681787", "sequence": "TGGGAGTCTAAGTCTCTTTTGATCACACTTTAAAGACCAAAAGGTAGAAGCGCAAAGACGTTATCTGTCCAATATTACAAACCTAGTAAGTGGTGGAATTTGGCCTTGAACCCAGATCTGTAACTCCAGAGCCGAAGTGCTTCACCCACCTCCCTGTGGTG"}, {"id": "186681788", "sequence": "G"}, {"id": "186681789", "sequence": "T"}, {"id": "186681790", "sequence": "TAT"}, {"id": "186681792", "sequence": "T"}, {"id": "186683069", "sequence": "G"}, {"id": "186683079", "sequence": "G"}, {"id": "186683080", "sequence": "TACCCCGGAATCCCTGCCGCGGCCCCTCGGGCCTGTCCACATCCCTCTGCCCCTCCCAGACCTCTGTCCTTCCACCAATCGCCTCCCGCAGCCCCGAGCCGCCACTCCCAGTCCCCCGAGTCCCTGCCGCGCGCCCTCGCGCCTGTCCACATCCCTCTGCCCATCCGAGACCTCTGTCCTTACACCACTAGCCACCCCACGTGGGACTTCCATGGCTTCTGAGTACAAGGCCAGCCCCCCGGCCCACCAGCTTTCGGAATGCCTGCTTACCTCTTTTTCTGTAGA"}, {"id": "186683081", "sequence": "CCGG"}, {"id": "186683083", "sequence": "C"}]} )"; - Graph source; - json2pb(source, graph_json.c_str(), graph_json.size()); - - VG vg; - vg.extend(source); + auto source_graph = json_to_graph(graph_json); bdsg::HashGraph extractor; @@ -1105,7 +1102,7 @@ TEST_CASE( "Connecting graph extraction works on a cool loop without leaving ext pos_t dest_pos = make_pos_t(186681787, true, 131); // If we have strict max length set to false, we may get extra tips. - unordered_map connect_trans = algorithms::extract_connecting_graph(&vg, &extractor, max_dist, src_pos, dest_pos, false); + unordered_map connect_trans = algorithms::extract_connecting_graph(source_graph.get(), &extractor, max_dist, src_pos, dest_pos, false); std::vector tip_handles = handlegraph::algorithms::find_tips(&extractor); // There ought to be at least the two tips REQUIRE(tip_handles.size() >= 2); @@ -1113,7 +1110,7 @@ TEST_CASE( "Connecting graph extraction works on a cool loop without leaving ext extractor.clear(); // If we have strict connecting-ness set to true, we won't get any extra tips. - connect_trans = algorithms::extract_connecting_graph(&vg, &extractor, max_dist, src_pos, dest_pos, true); + connect_trans = algorithms::extract_connecting_graph(source_graph.get(), &extractor, max_dist, src_pos, dest_pos, true); tip_handles = handlegraph::algorithms::find_tips(&extractor); // There ought to be just the two tips REQUIRE(tip_handles.size() == 2); @@ -1688,11 +1685,7 @@ TEST_CASE( "Connecting graph extraction works on a particular case without leavi )"; - Graph source; - json2pb(source, graph_json.c_str(), graph_json.size()); - - VG vg; - vg.extend(source); + auto vg = json_to_graph(graph_json); VG extractor; @@ -5385,11 +5378,7 @@ TEST_CASE("simplify_siblings() works on a graph with a reversing self loop", "[a {"edge": [{"from": "1", "to": "3"}, {"from": "1", "to": "2"}, {"from": "2", "to": "2", "to_end": true}], "node": [{"id": "1", "sequence": "T"}, {"id": "2", "sequence": "A"}, {"id": "3", "sequence": "ACA"}], "path": [{"mapping": [{"edit": [{"from_length": 1, "to_length": 1}], "position": {"node_id": "1"}, "rank": "1"}, {"edit": [{"from_length": 1, "to_length": 1}], "position": {"node_id": "2"}, "rank": "2"}, {"edit": [{"from_length": 1, "to_length": 1}], "position": {"is_reverse": true, "node_id": "2"}, "rank": "3"}], "name": "x"}, {"mapping": [{"edit": [{"from_length": 1, "to_length": 1}], "position": {"node_id": "1"}, "rank": "1"}, {"edit": [{"from_length": 3, "to_length": 3}], "position": {"node_id": "3"}, "rank": "2"}], "name": "y"}]} )"; - Graph source; - json2pb(source, graph_json.c_str(), graph_json.size()); - - VG graph; - graph.extend(source); + auto graph = json_to_graph(graph_json); @@ -5405,11 +5394,7 @@ TEST_CASE("simplify_siblings() works on a smaller graph with a reversing self lo {"edge": [{"from": "1", "to": "3"}, {"from": "1", "to": "2"}, {"from": "2", "to": "2", "to_end": true}], "node": [{"id": "1", "sequence": "T"}, {"id": "2", "sequence": "A"}, {"id": "3", "sequence": "A"}], "path": [{"mapping": [{"edit": [{"from_length": 1, "to_length": 1}], "position": {"node_id": "1"}, "rank": "1"}, {"edit": [{"from_length": 1, "to_length": 1}], "position": {"node_id": "2"}, "rank": "2"}, {"edit": [{"from_length": 1, "to_length": 1}], "position": {"is_reverse": true, "node_id": "2"}, "rank": "3"}], "name": "x"}]} )"; - Graph source; - json2pb(source, graph_json.c_str(), graph_json.size()); - - VG graph; - graph.extend(source); + auto graph = json_to_graph(graph_json); @@ -5425,11 +5410,7 @@ TEST_CASE("normalize() works on a graph with a reversing self loop", "[algorithm {"edge": [{"from": "1", "to": "3"}, {"from": "1", "to": "2"}, {"from": "2", "to": "2", "to_end": true}], "node": [{"id": "1", "sequence": "T"}, {"id": "2", "sequence": "A"}, {"id": "3", "sequence": "ACA"}], "path": [{"mapping": [{"edit": [{"from_length": 1, "to_length": 1}], "position": {"node_id": "1"}, "rank": "1"}, {"edit": [{"from_length": 1, "to_length": 1}], "position": {"node_id": "2"}, "rank": "2"}, {"edit": [{"from_length": 1, "to_length": 1}], "position": {"is_reverse": true, "node_id": "2"}, "rank": "3"}], "name": "x"}, {"mapping": [{"edit": [{"from_length": 1, "to_length": 1}], "position": {"node_id": "1"}, "rank": "1"}, {"edit": [{"from_length": 3, "to_length": 3}], "position": {"node_id": "3"}, "rank": "2"}], "name": "y"}]} )"; - Graph source; - json2pb(source, graph_json.c_str(), graph_json.size()); - - VG graph; - graph.extend(source); + auto graph = json_to_graph(graph_json); diff --git a/src/unittest/vpkg.cpp b/src/unittest/vpkg.cpp index 51a849c446..6e1ed8e4ab 100644 --- a/src/unittest/vpkg.cpp +++ b/src/unittest/vpkg.cpp @@ -14,6 +14,7 @@ #include "../vg.hpp" #include "../snarl_seed_clusterer.hpp" #include "vg/io/json2pb.h" +#include "support/json.hpp" #include #include #include @@ -49,13 +50,9 @@ TEST_CASE("We can read and write XG", "[vpkg][handlegraph][xg]") { "edge":[{"to":2,"from":1}]} )"; - // Load the JSON - Graph proto_graph; - json2pb(proto_graph, graph_json.c_str(), graph_json.size()); - - // Build the xg index + // Load the JSON and build the xg index xg::XG xg_index; - xg_index.from_path_handle_graph(VG(proto_graph)); + xg_index.from_path_handle_graph(*json_to_graph(graph_json)); stringstream ss; @@ -149,12 +146,8 @@ TEST_CASE("We can read VG from a VPKG-wrapped stream as a VG", "[vpkg][handlegra "edge":[{"to":2,"from":1}]} )"; - // Load the JSON - Graph proto_graph; - json2pb(proto_graph, graph_json.c_str(), graph_json.size()); - - // Build the VG - vg::VG vg_graph(proto_graph); + // Load the JSON and build the VG + auto vg_graph = json_to_graph(graph_json); // Save it stringstream ss; @@ -180,12 +173,8 @@ TEST_CASE("We can read VG from a VPKG-wrapped stream as a HandleGraph which is a "edge":[{"to":2,"from":1}]} )"; - // Load the JSON - Graph proto_graph; - json2pb(proto_graph, graph_json.c_str(), graph_json.size()); - - // Build the VG - vg::VG vg_graph(proto_graph); + // Load the JSON and build the VG + auto vg_graph = json_to_graph(graph_json); // Save it stringstream ss; @@ -211,12 +200,8 @@ TEST_CASE("We can read VG from a VPKG-wrapped stream as a HandleGraph which is a TEST_CASE("We can read an empty VG as a HandleGraph", "[vpkg][handlegraph][vg][empty]") { string graph_json = "{}"; - // Load the JSON - Graph proto_graph; - json2pb(proto_graph, graph_json.c_str(), graph_json.size()); - - // Build the VG - vg::VG vg_graph(proto_graph); + // Load the JSON and build the VG + auto vg_graph = json_to_graph(graph_json); // Save it stringstream ss; @@ -241,12 +226,8 @@ TEST_CASE("We prefer to read a graph as the first provided type that matches", " "edge":[{"to":2,"from":1}]} )"; - // Load the JSON - Graph proto_graph; - json2pb(proto_graph, graph_json.c_str(), graph_json.size()); - - // Build the VG - vg::VG vg_graph(proto_graph); + // Load the JSON and build the VG + auto vg_graph = json_to_graph(graph_json); // Save it stringstream ss; diff --git a/src/unittest/xdrop_aligner.cpp b/src/unittest/xdrop_aligner.cpp index f745b8f66a..f943b0bf3c 100644 --- a/src/unittest/xdrop_aligner.cpp +++ b/src/unittest/xdrop_aligner.cpp @@ -11,6 +11,7 @@ #include #include "test_aligner.hpp" #include "catch.hpp" +#include "support/json.hpp" #include "bdsg/hash_graph.hpp" namespace vg { @@ -765,11 +766,7 @@ TEST_CASE("XdropAligner doesn't crash on a case where it is hard to find a seed" string graph_json = R"({"edge": [{"from": "92345167", "to": "92345168"}, {"from": "92345182", "to": "92345183"}, {"from": "92345165", "to": "92345166"}, {"from": "92345177", "to": "92345178"}, {"from": "92345171", "to": "92345172"}, {"from": "92345161", "to": "92345162"}, {"from": "92345183", "to": "92345184"}, {"from": "92345181", "to": "92345182"}, {"from": "92345178", "to": "92345179"}, {"from": "92345166", "to": "92345167"}, {"from": "92345179", "to": "92345180"}, {"from": "92345173", "to": "92345174"}, {"from": "92345184", "to": "92345185"}, {"from": "92345169", "to": "92345170"}, {"from": "92345185", "to": "92345186"}, {"from": "92345160", "to": "92345161"}, {"from": "92345174", "to": "92345175"}, {"from": "92345162", "to": "92345163"}, {"from": "92345175", "to": "92345176"}, {"from": "92345168", "to": "92345169"}, {"from": "92345163", "to": "92345164"}, {"from": "92345172", "to": "92345173"}, {"from": "92345180", "to": "92345181"}, {"from": "92345176", "to": "92345177"}, {"from": "92345170", "to": "92345171"}, {"from": "92345164", "to": "92345165"}], "node": [{"id": "92345167", "sequence": "TTTATATATATATATTTATATATATATATTTA"}, {"id": "92345182", "sequence": "TATATATATTTATATATATATTTATATATATA"}, {"id": "92345165", "sequence": "ATATATATATATTTATATATATTTATATATTA"}, {"id": "92345177", "sequence": "TTTATATATATATTTATATATATATATTATAT"}, {"id": "92345171", "sequence": "TTATATATATATTTATATATATATTTATATAT"}, {"id": "92345161", "sequence": "ATATATTTATATATTTTTATATATTATATATT"}, {"id": "92345183", "sequence": "TTTATATATATTTATATATATATTTATATATA"}, {"id": "92345181", "sequence": "ATATATTATATATATATTTATATATATATTTA"}, {"id": "92345178", "sequence": "ATATATTTATATATATATTTATATATATATTT"}, {"id": "92345166", "sequence": "TTTATATATATTTATATATATATTTATATATA"}, {"id": "92345179", "sequence": "ATATATATATTTATATATATATTTATATATAT"}, {"id": "92345173", "sequence": "ATATTTATATATATATATTTATATATATATTT"}, {"id": "92345184", "sequence": "TATTTATATATATATTTATATATATTTATATA"}, {"id": "92345169", "sequence": "TTTATATATATATTTATATATATATTTATATA"}, {"id": "92345185", "sequence": "TATATTTATATATATATATATATATTTATATA"}, {"id": "92345160", "sequence": "ATTTATATATATATTTATATATATATTTATAT"}, {"id": "92345174", "sequence": "ATATATATATTTATATATATATTATTTATATA"}, {"id": "92345162", "sequence": "TATATATATATTTATATATTATATATATATTT"}, {"id": "92345175", "sequence": "TATATTTATATATATATTATATATATATTTAT"}, {"id": "92345168", "sequence": "TATATATATTTATATATATATTTATATATATA"}, {"id": "92345163", "sequence": "ATATATTTATATATATATTTATATATATTTAT"}, {"id": "92345172", "sequence": "ATATATATATATTTATATATATATTTATATAT"}, {"id": "92345180", "sequence": "ATTTATATATATATTTATATATATATTTATAT"}, {"id": "92345176", "sequence": "ATATATATATTATATATATATTTATATATATA"}, {"id": "92345170", "sequence": "TATATTTATATATATATATTATATATATATAT"}, {"id": "92345164", "sequence": "ATATATATTTATATATATTTATATATATATTT"}, {"id": "92345186", "sequence": "TATATTTATATATATTTATATATATATTTATA"}]})"; - Graph source; - json2pb(source, graph_json.c_str(), graph_json.size()); - - VG graph; - graph.extend(source); + auto graph = json_to_graph(graph_json); Alignment aln; aln.set_sequence("CAGCACTTTGGGAGGCCAAGGTGGGTGGATCATCTGAGGTCAGGAGTTTGAGACCAGCCTGACCAACATGGTGAAATCCTGTCTCTACTGAAAATACTAAAATTAGCCAGGCGTGGCGGCCAGTGCCTGTAATCCCGGCTACTGGGGAGG"); diff --git a/src/unittest/xg.cpp b/src/unittest/xg.cpp index d74db5d0b0..fbe0e8ff1e 100644 --- a/src/unittest/xg.cpp +++ b/src/unittest/xg.cpp @@ -9,6 +9,7 @@ #include "xg.hpp" #include "graph.hpp" #include "algorithms/subgraph.hpp" +#include "support/json.hpp" #include namespace vg { @@ -23,13 +24,9 @@ TEST_CASE("We can build an xg index on a nice graph", "[xg]") { "edge":[{"to":2,"from":1}]} )"; - // Load the JSON - Graph proto_graph; - json2pb(proto_graph, graph_json.c_str(), graph_json.size()); - - // Build the xg index + // Load the JSON and build the xg index xg::XG xg_index; - xg_index.from_path_handle_graph(VG(proto_graph)); + xg_index.from_path_handle_graph(*json_to_graph(graph_json)); VG vg_graph; algorithms::extract_context(xg_index, vg_graph, xg_index.get_handle(1), 0, 100); @@ -50,13 +47,9 @@ TEST_CASE("We can build an xg index on a nasty graph", "[xg]") { "edge":[{"to":2,"from":1}]} )"; - // Load the JSON - Graph proto_graph; - json2pb(proto_graph, graph_json.c_str(), graph_json.size()); - - // Build the xg index + // Load the JSON and build the xg index xg::XG xg_index; - xg_index.from_path_handle_graph(VG(proto_graph)); + xg_index.from_path_handle_graph(*json_to_graph(graph_json)); VG vg_graph; algorithms::extract_context(xg_index, vg_graph, xg_index.get_handle(1), 0, 100); @@ -169,7 +162,7 @@ TEST_CASE("We can build an xg index on a very nasty graph", "[xg]") { sort_by_id_dedup_and_clean(proto_graph); // Build the xg index xg::XG xg_index; - xg_index.from_path_handle_graph(VG(proto_graph)); + xg_index.from_path_handle_graph(VG(proto_graph)); // TODO: replace with json_to_graph() once proto_graph usage is refactored SECTION("Context extraction gets something") { VG graph; @@ -306,7 +299,7 @@ TEST_CASE("We can build the xg index on a small graph with discontinuous node id sort_by_id_dedup_and_clean(proto_graph); // Build the xg index xg::XG xg_index; - xg_index.from_path_handle_graph(VG(proto_graph)); + xg_index.from_path_handle_graph(VG(proto_graph)); // TODO: replace with json_to_graph() once proto_graph usage is refactored VG vg_graph; algorithms::extract_context(xg_index, vg_graph, xg_index.get_handle(10), 0, 100); @@ -327,13 +320,9 @@ TEST_CASE("Looping over XG handles in parallel works", "[xg]") { "edge":[{"to":2,"from":1}]} )"; - // Load the JSON - Graph proto_graph; - json2pb(proto_graph, graph_json.c_str(), graph_json.size()); - - // Build the xg index + // Load the JSON and build the xg index xg::XG xg_index; - xg_index.from_path_handle_graph(VG(proto_graph)); + xg_index.from_path_handle_graph(*json_to_graph(graph_json)); size_t count = 0; From 2c3721db6e2174004bd4037a9c61c413a8753966 Mon Sep 17 00:00:00 2001 From: Adam Novak Date: Tue, 10 Feb 2026 15:14:32 -0500 Subject: [PATCH 19/77] Revert "Commit partial synthetic refactor to use new JSON load method" This reverts commit 5436d73117a8c5c30e57dd359ced458a8ee6f693. --- src/unittest/cactus.cpp | 21 ++-- src/unittest/chunker.cpp | 9 +- src/unittest/copy_graph.cpp | 89 +++++++++------- src/unittest/dijkstra.cpp | 14 +-- src/unittest/gbwt_extender.cpp | 6 +- src/unittest/genotypekit.cpp | 57 +++++----- src/unittest/genotyper.cpp | 9 +- src/unittest/haplotypes.cpp | 26 +++-- src/unittest/indexed_vg.cpp | 1 - src/unittest/mapper.cpp | 32 ++++-- src/unittest/minimizer_mapper.cpp | 23 +++-- src/unittest/multipath_alignment_graph.cpp | 7 +- src/unittest/multipath_mapper.cpp | 25 +++-- src/unittest/path_component_index.cpp | 9 +- src/unittest/phase_unfolder.cpp | 33 ++++-- src/unittest/readfilter.cpp | 6 +- src/unittest/snarl_distance_index.cpp | 37 +++++-- src/unittest/snarls.cpp | 49 ++++++--- src/unittest/variant_adder.cpp | 115 ++++++++++++++------- src/unittest/vg.cpp | 7 +- src/unittest/vg_algorithms.cpp | 35 +++++-- src/unittest/vpkg.cpp | 41 ++++++-- src/unittest/xdrop_aligner.cpp | 7 +- src/unittest/xg.cpp | 29 ++++-- 24 files changed, 457 insertions(+), 230 deletions(-) diff --git a/src/unittest/cactus.cpp b/src/unittest/cactus.cpp index 89a2f95cb8..7447ee247d 100644 --- a/src/unittest/cactus.cpp +++ b/src/unittest/cactus.cpp @@ -8,7 +8,6 @@ #include "vg/io/json2pb.h" #include "../cactus.hpp" #include "catch.hpp" -#include "support/json.hpp" namespace vg { namespace unittest { @@ -16,6 +15,8 @@ using namespace std; TEST_CASE("We can convert a two-tailed graph to Cactus", "[cactus]") { + VG graph; + string graph_json = R"( {"node":[{"sequence":"GT","id":7575}, {"sequence":"TGTTAACAGCACAACATTTA","id":7580}, @@ -25,15 +26,19 @@ TEST_CASE("We can convert a two-tailed graph to Cactus", "[cactus]") { {"from":7575,"to":7576}]} )"; - auto graph = json_to_graph(graph_json); + Graph g; + json2pb(g, graph_json.c_str(), graph_json.size()); + graph.extend(g); - // Make sure we can make a Cactus graph and get something out. - auto cactusified = cactusify(*graph); + // Make sure we can make a Cactus graph and get something out. + auto cactusified = cactusify(graph); REQUIRE(cactusified.is_valid()); } TEST_CASE("We can convert a hairpin graph to Cactus", "[cactus]") { + VG graph; + // Here's a graph where only the left side of node 2 is dangling, and the right side of node 1 has a self loop. string graph_json = R"( {"node": [{"sequence": "A", "id": 1}, @@ -42,10 +47,12 @@ TEST_CASE("We can convert a hairpin graph to Cactus", "[cactus]") { {"from": 1, "to": 1, "to_end": true}]} )"; - auto graph = json_to_graph(graph_json); + Graph g; + json2pb(g, graph_json.c_str(), graph_json.size()); + graph.extend(g); - // Make sure we can make a Cactus graph and get something out. - auto cactusified = cactusify(*graph); + // Make sure we can make a Cactus graph and get something out. + auto cactusified = cactusify(graph); REQUIRE(cactusified.is_valid()); } diff --git a/src/unittest/chunker.cpp b/src/unittest/chunker.cpp index 81b136fe91..24f7d3b645 100644 --- a/src/unittest/chunker.cpp +++ b/src/unittest/chunker.cpp @@ -7,7 +7,6 @@ #include "vg.hpp" #include "xg.hpp" #include "path.hpp" -#include "support/json.hpp" namespace vg { namespace unittest { @@ -84,9 +83,13 @@ TEST_CASE("basic graph chunking", "[chunk]") { )"; - // Load it and pass it over to XG + // Load it into Protobuf + Graph chunk; + json2pb(chunk, graph_json.c_str(), graph_json.size()); + + // Pass it over to XG xg::XG index; - index.from_path_handle_graph(*json_to_graph(graph_json)); + index.from_path_handle_graph(VG(chunk)); PathChunker chunker(&index); diff --git a/src/unittest/copy_graph.cpp b/src/unittest/copy_graph.cpp index 6d83c36fda..581b683130 100644 --- a/src/unittest/copy_graph.cpp +++ b/src/unittest/copy_graph.cpp @@ -2,7 +2,6 @@ #include "../handle.hpp" #include "../vg.hpp" #include "xg.hpp" -#include "support/json.hpp" #include "bdsg/packed_graph.hpp" #include "bdsg/hash_graph.hpp" @@ -54,13 +53,14 @@ namespace vg { ] } )"; - auto graph_ptr = json_to_graph(graph_json); - + Graph proto_graph; + json2pb(proto_graph, graph_json.c_str(), graph_json.size()); + xg::XG xg; - xg.from_path_handle_graph(*graph_ptr); + xg.from_path_handle_graph(VG(proto_graph)); VG vg; handlealgs::copy_handle_graph(&xg, &vg); - + REQUIRE(xg.get_node_count() == 1); REQUIRE(vg.get_node_count() == 1); } @@ -72,13 +72,14 @@ namespace vg { ] } )"; - auto graph_ptr = json_to_graph(graph_json); - + Graph proto_graph; + json2pb(proto_graph, graph_json.c_str(), graph_json.size()); + xg::XG xg; - xg.from_path_handle_graph(*graph_ptr); + xg.from_path_handle_graph(VG(proto_graph)); bdsg::PackedGraph pg; handlealgs::copy_handle_graph(&xg, &pg); - + REQUIRE(xg.get_node_count() == 1); REQUIRE(pg.get_node_count() == 1); } @@ -90,10 +91,11 @@ namespace vg { ] } )"; - auto graph_ptr = json_to_graph(graph_json); - + Graph proto_graph; + json2pb(proto_graph, graph_json.c_str(), graph_json.size()); + xg::XG xg; - xg.from_path_handle_graph(*graph_ptr); + xg.from_path_handle_graph(VG(proto_graph)); bdsg::HashGraph hg; handlealgs::copy_handle_graph(&xg, &hg); @@ -118,10 +120,11 @@ namespace vg { ] } )"; - auto graph_ptr = json_to_graph(graph_json); - + Graph proto_graph; + json2pb(proto_graph, graph_json.c_str(), graph_json.size()); + xg::XG xg; - xg.from_path_handle_graph(*graph_ptr); + xg.from_path_handle_graph(VG(proto_graph)); VG vg; handlealgs::copy_handle_graph(&xg, &vg); @@ -148,10 +151,11 @@ namespace vg { ] } )"; - auto graph_ptr = json_to_graph(graph_json); - + Graph proto_graph; + json2pb(proto_graph, graph_json.c_str(), graph_json.size()); + xg::XG xg; - xg.from_path_handle_graph(*graph_ptr); + xg.from_path_handle_graph(VG(proto_graph)); bdsg::PackedGraph pg; handlealgs::copy_handle_graph(&xg, &pg); @@ -190,10 +194,11 @@ namespace vg { ] } )"; - auto graph_ptr = json_to_graph(graph_json); - + Graph proto_graph; + json2pb(proto_graph, graph_json.c_str(), graph_json.size()); + xg::XG xg; - xg.from_path_handle_graph(*graph_ptr); + xg.from_path_handle_graph(VG(proto_graph)); bdsg::HashGraph hg; handlealgs::copy_handle_graph(&xg, &hg); @@ -234,10 +239,11 @@ namespace vg { ] } )"; - auto graph_ptr = json_to_graph(graph_json); - + Graph proto_graph; + json2pb(proto_graph, graph_json.c_str(), graph_json.size()); + xg::XG xg; - xg.from_path_handle_graph(*graph_ptr); + xg.from_path_handle_graph(VG(proto_graph)); VG vg; handlealgs::copy_handle_graph(&xg, &vg); @@ -268,10 +274,11 @@ namespace vg { ] } )"; - auto graph_ptr = json_to_graph(graph_json); - + Graph proto_graph; + json2pb(proto_graph, graph_json.c_str(), graph_json.size()); + xg::XG xg; - xg.from_path_handle_graph(*graph_ptr); + xg.from_path_handle_graph(VG(proto_graph)); bdsg::PackedGraph pg; handlealgs::copy_handle_graph(&xg, &pg); @@ -314,10 +321,11 @@ namespace vg { ] } )"; - auto graph_ptr = json_to_graph(graph_json); - + Graph proto_graph; + json2pb(proto_graph, graph_json.c_str(), graph_json.size()); + xg::XG xg; - xg.from_path_handle_graph(*graph_ptr); + xg.from_path_handle_graph(VG(proto_graph)); bdsg::HashGraph hg; handlealgs::copy_handle_graph(&xg, &hg); @@ -374,10 +382,11 @@ namespace vg { ] } )"; - auto graph_ptr = json_to_graph(graph_json); - + Graph proto_graph; + json2pb(proto_graph, graph_json.c_str(), graph_json.size()); + xg::XG xg; - xg.from_path_handle_graph(*graph_ptr); + xg.from_path_handle_graph(VG(proto_graph)); VG vg; handlealgs::copy_path_handle_graph(&xg, &vg); @@ -435,10 +444,11 @@ namespace vg { ] } )"; - auto graph_ptr = json_to_graph(graph_json); - + Graph proto_graph; + json2pb(proto_graph, graph_json.c_str(), graph_json.size()); + xg::XG xg; - xg.from_path_handle_graph(*graph_ptr); + xg.from_path_handle_graph(VG(proto_graph)); bdsg::PackedGraph pg; handlealgs::copy_path_handle_graph(&xg, &pg); @@ -511,10 +521,11 @@ namespace vg { ] } )"; - auto graph_ptr = json_to_graph(graph_json); - + Graph proto_graph; + json2pb(proto_graph, graph_json.c_str(), graph_json.size()); + xg::XG xg; - xg.from_path_handle_graph(*graph_ptr); + xg.from_path_handle_graph(VG(proto_graph)); bdsg::HashGraph hg; handlealgs::copy_path_handle_graph(&xg, &hg); diff --git a/src/unittest/dijkstra.cpp b/src/unittest/dijkstra.cpp index e3d09743b7..2608567153 100644 --- a/src/unittest/dijkstra.cpp +++ b/src/unittest/dijkstra.cpp @@ -9,7 +9,6 @@ #include "vg/io/json2pb.h" #include "../vg.hpp" #include "catch.hpp" -#include "support/json.hpp" #include @@ -129,27 +128,30 @@ TEST_CASE("Dijkstra search works on a particular problem graph", "[dijkstra][alg {"node":[{"sequence":"A","id":"2454530"},{"sequence":"AGTGCTGGAGAGGATGTGGAGAAATAGGAAC","id":"2454529"},{"sequence":"C","id":"2454532"},{"sequence":"TTTTACACTGTTGGTGGGACTGTAAA","id":"2454533"},{"sequence":"A","id":"2454527"},{"sequence":"C","id":"2454528"},{"sequence":"G","id":"2454531"},{"sequence":"C","id":"2454534"},{"sequence":"T","id":"2454535"},{"sequence":"GGGTAATAA","id":"2454526"},{"sequence":"TAGTTCAACCATTGTGGAAGACTGTGGCAATT","id":"2454536"}],"edge":[{"from":"2454530","to":"2454532"},{"from":"2454530","to":"2454533"},{"from":"2454529","to":"2454530"},{"from":"2454529","to":"2454531"},{"from":"2454532","to":"2454533"},{"from":"2454533","to":"2454534"},{"from":"2454533","to":"2454535"},{"from":"2454527","to":"2454529"},{"from":"2454528","to":"2454529"},{"from":"2454531","to":"2454532"},{"from":"2454531","to":"2454533"},{"from":"2454534","to":"2454536"},{"from":"2454535","to":"2454536"},{"from":"2454526","to":"2454527"},{"from":"2454526","to":"2454528"}],"path":[{"name":"21","mapping":[{"position":{"node_id":"2454526"},"edit":[{"from_length":9,"to_length":9}],"rank":"3049077"},{"position":{"node_id":"2454528"},"edit":[{"from_length":1,"to_length":1}],"rank":"3049078"},{"position":{"node_id":"2454529"},"edit":[{"from_length":31,"to_length":31}],"rank":"3049079"},{"position":{"node_id":"2454531"},"edit":[{"from_length":1,"to_length":1}],"rank":"3049080"},{"position":{"node_id":"2454532"},"edit":[{"from_length":1,"to_length":1}],"rank":"3049081"},{"position":{"node_id":"2454533"},"edit":[{"from_length":26,"to_length":26}],"rank":"3049082"},{"position":{"node_id":"2454535"},"edit":[{"from_length":1,"to_length":1}],"rank":"3049083"},{"position":{"node_id":"2454536"},"edit":[{"from_length":32,"to_length":32}],"rank":"3049084"}]}]} )"; + Graph g; + json2pb(g, graph_json); + // Wrap the graph in a HandleGraph - auto graph = json_to_graph(graph_json); + VG graph(g); // Decide where to start - handle_t start = graph->get_handle(2454536, true); + handle_t start = graph.get_handle(2454536, true); // Track what we reach and at what distance unordered_map seen; - handlealgs::dijkstra(graph.get(), start, [&](const handle_t& reached, size_t distance) { + handlealgs::dijkstra(&graph, start, [&](const handle_t& reached, size_t distance) { seen[reached] = distance; return true; }); - REQUIRE(seen.size() == graph->get_node_count()); + REQUIRE(seen.size() == graph.get_node_count()); } TEST_CASE( "Shortest path through chain with loop", "[dijkstra][algorithms]" ) { - bdsg::HashGraph graph; + VG graph; handle_t n1 = graph.create_handle("GCA"); handle_t n2 = graph.create_handle("T"); diff --git a/src/unittest/gbwt_extender.cpp b/src/unittest/gbwt_extender.cpp index b8b24710dd..d04a225fdb 100644 --- a/src/unittest/gbwt_extender.cpp +++ b/src/unittest/gbwt_extender.cpp @@ -8,7 +8,6 @@ #include "vg/io/json2pb.h" #include "../utility.hpp" #include "../vg.hpp" -#include "support/json.hpp" #include @@ -91,8 +90,9 @@ gbwt::GBWT build_gbwt_index() { // Build a GBWTGraph using the provided GBWT index. gbwtgraph::GBWTGraph build_gbwt_graph(const gbwt::GBWT& gbwt_index) { - auto vg_graph_ptr = json_to_graph(gapless_extender_graph); - auto& vg_graph = *vg_graph_ptr; + Graph graph; + json2pb(graph, gapless_extender_graph.c_str(), gapless_extender_graph.size()); + VG vg_graph(graph); return gbwtgraph::GBWTGraph(gbwt_index, vg_graph, nullptr); } diff --git a/src/unittest/genotypekit.cpp b/src/unittest/genotypekit.cpp index 31d535dca5..af9bc2a4d8 100644 --- a/src/unittest/genotypekit.cpp +++ b/src/unittest/genotypekit.cpp @@ -10,7 +10,6 @@ #include "../traversal_finder.hpp" #include "xg.hpp" #include "../haplotype_extracter.hpp" -#include "support/json.hpp" namespace Catch { @@ -106,9 +105,11 @@ TEST_CASE("sites can be found with Cactus", "[genotype]") { )"; // Make an actual graph - auto graph_ptr = json_to_graph(graph_json); - VG& graph = *dynamic_cast(graph_ptr.get()); - + VG graph; + Graph chunk; + json2pb(chunk, graph_json.c_str(), graph_json.size()); + graph.merge(chunk); + // Make a CactusSnarlFinder unique_ptr finder(new CactusSnarlFinder(graph)); @@ -238,30 +239,32 @@ TEST_CASE("sites can be found with the IntegratedSnarlFinder", "[genotype][integ )"; // Make an actual graph - auto graph_ptr = json_to_graph(graph_json); - VG& graph = *dynamic_cast(graph_ptr.get()); - + VG graph; + Graph chunk; + json2pb(chunk, graph_json.c_str(), graph_json.size()); + graph.merge(chunk); + // Make an IntegratedSnarlFinder unique_ptr finder(new IntegratedSnarlFinder(graph)); - + SECTION("IntegratedSnarlFinder should find two top-level sites") { - + SnarlManager manager = finder->find_snarls(); - + auto sites = manager.top_level_snarls(); - + REQUIRE(sites.size() == 2); - + // Order them const Snarl* site_1 = sites[0]->start().node_id() > sites[1]->start().node_id() ? sites[1] : sites[0]; const Snarl* site_2 = sites[0]->start().node_id() > sites[1]->start().node_id() ? sites[0] : sites[1]; - + SECTION("the first site should be 1 fwd to 6 fwd") { REQUIRE(site_1->start().node_id() == 1); REQUIRE(site_1->start().backward() == false); REQUIRE(site_1->end().node_id() == 6); REQUIRE(site_1->end().backward() == false); - + SECTION("and should contain exactly nodes 1 through 6") { auto nodes = manager.deep_contents(site_1, graph, true).first; set correct{graph.get_node(1), graph.get_node(2), @@ -589,9 +592,11 @@ TEST_CASE("CactusSnarlFinder safely handles a single node graph", "[genotype][ca )"; // Make an actual graph - auto graph_ptr = json_to_graph(graph_json); - VG& graph = *dynamic_cast(graph_ptr.get()); - + VG graph; + Graph chunk; + json2pb(chunk, graph_json.c_str(), graph_json.size()); + graph.merge(chunk); + // Make a CactusSnarlFinder unique_ptr finder(new CactusSnarlFinder(graph)); @@ -1114,9 +1119,11 @@ TEST_CASE("CactusSnarlFinder throws an error instead of crashing when the graph )"; // Make an actual graph - auto graph_ptr = json_to_graph(graph_json); - VG& graph = *dynamic_cast(graph_ptr.get()); - + VG graph; + Graph chunk; + json2pb(chunk, graph_json.c_str(), graph_json.size()); + graph.merge(chunk); + // Make a CactusSnarlFinder unique_ptr finder(new CactusSnarlFinder(graph)); @@ -1216,9 +1223,11 @@ TEST_CASE("TrivialTraversalFinder can find traversals", "[genotype]") { )"; // Make an actual graph - auto graph_ptr = json_to_graph(graph_json); - auto& graph = *graph_ptr; - + VG graph; + Graph chunk; + json2pb(chunk, graph_json.c_str(), graph_json.size()); + graph.merge(chunk); + // Make a site Snarl site; site.mutable_start()->set_node_id(2); @@ -1226,7 +1235,7 @@ TEST_CASE("TrivialTraversalFinder can find traversals", "[genotype]") { site.set_type(ULTRABUBBLE); site.set_start_end_reachable(true); site.set_directed_acyclic_net_graph(true); - + // Make the TraversalFinder TraversalFinder* finder = new TrivialTraversalFinder(graph); diff --git a/src/unittest/genotyper.cpp b/src/unittest/genotyper.cpp index 19fa7eafd8..e2e9f7a142 100644 --- a/src/unittest/genotyper.cpp +++ b/src/unittest/genotyper.cpp @@ -7,7 +7,6 @@ #include "../snarls.hpp" #include "../cactus_snarl_finder.hpp" #include "../traversal_finder.hpp" -#include "support/json.hpp" namespace vg { namespace unittest { @@ -57,9 +56,11 @@ TEST_CASE("traversals can be found from reads", "[genotyper]") { )"; // Make an actual graph - auto graph_ptr = json_to_graph(graph_json); - VG& graph = *dynamic_cast(graph_ptr.get()); - + VG graph; + Graph chunk; + json2pb(chunk, graph_json.c_str(), graph_json.size()); + graph.merge(chunk); + // Find the snarls SnarlManager manager = CactusSnarlFinder(graph).find_snarls(); diff --git a/src/unittest/haplotypes.cpp b/src/unittest/haplotypes.cpp index fdbb80756f..e441bbe197 100644 --- a/src/unittest/haplotypes.cpp +++ b/src/unittest/haplotypes.cpp @@ -6,11 +6,9 @@ #include "haplotypes.hpp" #include "xg.hpp" #include "vg.hpp" -#include "support/json.hpp" #include -namespace vg { namespace unittest { using namespace std; @@ -95,14 +93,18 @@ TEST_CASE("We can represent appropriate graphs according to linear reference", " thread_t del_ref_thread = {tm[1], tm[2], tm[4]}; thread_t del_thread = {tm[1], tm[4]}; + vg::Graph SNP_proto_graph; + json2pb(SNP_proto_graph, SNP_graph_json.c_str(), SNP_graph_json.size()); // Build the xg index xg::XG SNP_xg_index; - SNP_xg_index.from_path_handle_graph(*json_to_graph(SNP_graph_json)); + SNP_xg_index.from_path_handle_graph(vg::VG(SNP_proto_graph)); vg::path_handle_t SNP_ref_path_handle = SNP_xg_index.get_path_handle("reference"); + vg::Graph del_proto_graph; + json2pb(del_proto_graph, del_graph_json.c_str(), del_graph_json.size()); // Build the xg index xg::XG del_xg_index; - del_xg_index.from_path_handle_graph(*json_to_graph(del_graph_json)); + del_xg_index.from_path_handle_graph(vg::VG(del_proto_graph)); vg::path_handle_t del_ref_path_handle = del_xg_index.get_path_handle("reference"); // NEGATIVE SNVs @@ -157,14 +159,18 @@ TEST_CASE("We can represent appropriate graphs according to linear reference", " thread_t double_thread = {tm[1], tm[2], tm[4]}; + vg::Graph long_proto_graph; + json2pb(long_proto_graph, long_graph_json.c_str(), long_graph_json.size()); // Build the xg index xg::XG long_xg_index; - long_xg_index.from_path_handle_graph(*json_to_graph(long_graph_json)); + long_xg_index.from_path_handle_graph(vg::VG(long_proto_graph)); vg::path_handle_t long_ref_path_handle = long_xg_index.get_path_handle("reference"); + vg::Graph double_proto_graph; + json2pb(double_proto_graph, double_graph_json.c_str(), double_graph_json.size()); // Build the xg index xg::XG double_xg_index; - double_xg_index.from_path_handle_graph(*json_to_graph(double_graph_json)); + double_xg_index.from_path_handle_graph(vg::VG(double_proto_graph)); vg::path_handle_t double_ref_path_handle = double_xg_index.get_path_handle("reference"); string matching_test_file = "matching_test.slls"; @@ -377,9 +383,12 @@ TEST_CASE("We can recognize a required crossover", "[hapo-score][gbwt]") { // This graph is the start of xy2 from test/small string graph_json = R"({"node": [{"id": 1, "sequence": "CAAATAAGGCTT"}, {"id": 2, "sequence": "G"}, {"id": 3, "sequence": "GGAAATTTTC"}, {"id": 4, "sequence": "C"}, {"id": 5, "sequence": "TGGAGTTCTATTATATTCC"}, {"id": 6, "sequence": "G"}, {"id": 7, "sequence": "A"}, {"id": 8, "sequence": "ACTCTCTGGTTCCTG"}, {"id": 9, "sequence": "A"}, {"id": 10, "sequence": "G"}, {"id": 11, "sequence": "TGCTATGTGTAACTAGTAATGGTAATGGATATGTTGGGCTTTTTTCTTTGATTTATTTGAAGTGACGTTTGACAATCTATCACTAGGGGTAATGTGGGGAAATGGAAAGAATACAAGATTTGGAGCCA"}], "edge": [{"from": 1, "to": 2}, {"from": 1, "to": 3}, {"from": 2, "to": 3}, {"from": 3, "to": 4}, {"from": 3, "to": 5}, {"from": 4, "to": 5}, {"from": 5, "to": 6}, {"from": 5, "to": 7}, {"from": 6, "to": 8}, {"from": 7, "to": 8}, {"from": 8, "to": 9}, {"from": 8, "to": 10}, {"from": 9, "to": 11}, {"from": 10, "to": 11}]})"; - // Load the JSON and build the xg index + // Load the JSON + vg::Graph proto_graph; + json2pb(proto_graph, graph_json.c_str(), graph_json.size()); + // Build the xg index xg::XG xg_index; - xg_index.from_path_handle_graph(*json_to_graph(graph_json)); + xg_index.from_path_handle_graph(vg::VG(proto_graph)); gbwt::Verbosity::set(gbwt::Verbosity::SILENT); gbwt::DynamicGBWT* gbwt_index = new gbwt::DynamicGBWT; @@ -471,4 +480,3 @@ TEST_CASE("We can recognize a required crossover", "[hapo-score][gbwt]") { } } -} diff --git a/src/unittest/indexed_vg.cpp b/src/unittest/indexed_vg.cpp index 1f1a687abc..7f74d92193 100644 --- a/src/unittest/indexed_vg.cpp +++ b/src/unittest/indexed_vg.cpp @@ -10,7 +10,6 @@ #include "../utility.hpp" #include "../algorithms/id_sort.hpp" #include "support/random_graph.hpp" -#include "support/json.hpp" #include "catch.hpp" namespace vg { diff --git a/src/unittest/mapper.cpp b/src/unittest/mapper.cpp index dcd830a7ed..2caf42d076 100644 --- a/src/unittest/mapper.cpp +++ b/src/unittest/mapper.cpp @@ -10,7 +10,6 @@ #include "xg.hpp" #include "../build_index.hpp" #include "catch.hpp" -#include "support/json.hpp" #include "../algorithms/alignment_path_offsets.hpp" namespace vg { @@ -27,8 +26,13 @@ TEST_CASE( "Mapper can map to a one-node graph", "[mapping][mapper]" ) { ] })"; - // Load the JSON and make it into a VG - auto graph = json_to_graph(graph_json); + // Load the JSON + Graph proto_graph; + json2pb(proto_graph, graph_json.c_str(), graph_json.size()); + + // Make it into a VG + VG graph; + graph.extend(proto_graph); // Make GCSA quiet gcsa::Verbosity::set(gcsa::Verbosity::SILENT); @@ -242,8 +246,13 @@ TEST_CASE( "Mapper finds optimal mapping for read starting with node-border MEM" {"position":{"node_id":1445},"rank":1060}]}]} )"; - // Load the JSON and make it into a VG - auto graph = json_to_graph(graph_json); + // Load the JSON + Graph proto_graph; + json2pb(proto_graph, graph_json.c_str(), graph_json.size()); + + // Make it into a VG + VG graph; + graph.extend(proto_graph); // Make GCSA quiet gcsa::Verbosity::set(gcsa::Verbosity::SILENT); @@ -303,8 +312,13 @@ TEST_CASE( "Mapper can annotate positions correctly on both strands", "[mapper][ ]} )"; - // Load the JSON and make it into a VG - auto graph = json_to_graph(graph_json); + // Load the JSON + Graph proto_graph; + json2pb(proto_graph, graph_json.c_str(), graph_json.size()); + + // Make it into a VG + VG graph; + graph.extend(proto_graph); // Make GCSA quiet gcsa::Verbosity::set(gcsa::Verbosity::SILENT); @@ -314,11 +328,11 @@ TEST_CASE( "Mapper can annotate positions correctly on both strands", "[mapper][ gcsa::LCPArray* lcpidx = nullptr; // Build the GCSA index - build_gcsa_lcp(*graph, gcsaidx, lcpidx, 16, 3); + build_gcsa_lcp(graph, gcsaidx, lcpidx, 16, 3); // Build the xg index xg::XG xg_index; - xg_index.from_path_handle_graph(*graph); + xg_index.from_path_handle_graph(graph); // Make a multipath mapper to map against the graph. Mapper mapper(&xg_index, gcsaidx, lcpidx); diff --git a/src/unittest/minimizer_mapper.cpp b/src/unittest/minimizer_mapper.cpp index aab6a8fedc..e2e25db870 100644 --- a/src/unittest/minimizer_mapper.cpp +++ b/src/unittest/minimizer_mapper.cpp @@ -7,7 +7,6 @@ #include "../io/json2graph.hpp" #include #include "../minimizer_mapper.hpp" -#include "support/json.hpp" #include "../build_index.hpp" #include "../integrated_snarl_finder.hpp" #include "../gbwt_extender.hpp" @@ -453,7 +452,9 @@ TEST_CASE("MinimizerMapper can map an empty string between odd points", "[giraff })"; // TODO: Write a json_to_handle_graph - auto graph = json_to_graph(graph_json); + vg::Graph proto_graph; + json2pb(proto_graph, graph_json.c_str(), graph_json.size()); + auto graph = vg::VG(proto_graph); Alignment aln; aln.set_sequence(""); @@ -461,7 +462,7 @@ TEST_CASE("MinimizerMapper can map an empty string between odd points", "[giraff pos_t left_anchor {55511921, false, 5}; // This is on the final base of the node pos_t right_anchor {55511925, false, 6}; - TestMinimizerMapper::align_sequence_between(left_anchor, right_anchor, 100, 20, graph.get(), &aligner, aln); + TestMinimizerMapper::align_sequence_between(left_anchor, right_anchor, 100, 20, &graph, &aligner, aln); // Make sure we get the right alignment. We should see the last base of '21 and go '21 to '24 to '25 and delete everything REQUIRE(aln.path().mapping_size() == 3); @@ -493,7 +494,9 @@ TEST_CASE("MinimizerMapper can map with an initial deletion", "[giraffe][mapping })"; // TODO: Write a json_to_handle_graph - auto graph = json_to_graph(graph_json); + vg::Graph proto_graph; + json2pb(proto_graph, graph_json.c_str(), graph_json.size()); + auto graph = vg::VG(proto_graph); Alignment aln; aln.set_sequence("CATTAG"); @@ -538,7 +541,9 @@ TEST_CASE("MinimizerMapper can map with an initial deletion on a multi-base node })"; // TODO: Write a json_to_handle_graph - auto graph = json_to_graph(graph_json); + vg::Graph proto_graph; + json2pb(proto_graph, graph_json.c_str(), graph_json.size()); + auto graph = vg::VG(proto_graph); Alignment aln; aln.set_sequence("CATTAG"); @@ -583,7 +588,9 @@ TEST_CASE("MinimizerMapper can map right off the past-the-end base", "[giraffe][ })"; // TODO: Write a json_to_handle_graph - auto graph = json_to_graph(graph_json); + vg::Graph proto_graph; + json2pb(proto_graph, graph_json.c_str(), graph_json.size()); + auto graph = vg::VG(proto_graph); Alignment aln; aln.set_sequence("CATTAG"); @@ -634,7 +641,9 @@ TEST_CASE("MinimizerMapper can find a significant indel instead of a tempting so })"; // TODO: Write a json_to_handle_graph - auto graph = json_to_graph(graph_json); + vg::Graph proto_graph; + json2pb(proto_graph, graph_json.c_str(), graph_json.size()); + auto graph = vg::VG(proto_graph); Alignment aln; aln.set_sequence("TTGAAAACCTGATATGTCTTATTTTTCTAACTATGGAATTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTGAGACGGAGTCTCGCTCTGTCGCCCAGGCTGGAGTGCAGTGGCGCGATCTCGGCTCACTGCAAGCTCCGCCTCCCGGGTTCACGCCATTCTCCTGCCTCAGCCTCCCGAGTAGCTGGGACTACAGGCGCCCGCTACCACGCCCGGCTAATTTTTTGTATTTTTTTT"); diff --git a/src/unittest/multipath_alignment_graph.cpp b/src/unittest/multipath_alignment_graph.cpp index 815d835638..bea5f687aa 100644 --- a/src/unittest/multipath_alignment_graph.cpp +++ b/src/unittest/multipath_alignment_graph.cpp @@ -11,7 +11,6 @@ #include "../snarl_distance_index.hpp" #include "catch.hpp" #include "support/test_aligner.hpp" -#include "support/json.hpp" @@ -48,8 +47,12 @@ TEST_CASE( "MultipathAlignmentGraph::align handles tails correctly", "[multipath })"; // Load the JSON + Graph proto_graph; + json2pb(proto_graph, graph_json.c_str(), graph_json.size()); + // Make it into a VG - auto vg = json_to_graph(graph_json); + VG vg; + vg.extend(proto_graph); // Make snarls on it CactusSnarlFinder bubble_finder(vg); diff --git a/src/unittest/multipath_mapper.cpp b/src/unittest/multipath_mapper.cpp index 1113a22202..be6d3b6194 100644 --- a/src/unittest/multipath_mapper.cpp +++ b/src/unittest/multipath_mapper.cpp @@ -10,7 +10,6 @@ #include "xg.hpp" #include "vg.hpp" #include "catch.hpp" -#include "support/json.hpp" namespace vg { namespace unittest { @@ -123,8 +122,12 @@ TEST_CASE( "MultipathMapper::query_cluster_graphs works", "[multipath][mapping][ })"; // Load the JSON + Graph proto_graph; + json2pb(proto_graph, graph_json.c_str(), graph_json.size()); + // Make it into a VG - auto graph = json_to_graph(graph_json); + VG graph; + graph.extend(proto_graph); // Make GCSA quiet gcsa::Verbosity::set(gcsa::Verbosity::SILENT); @@ -272,8 +275,12 @@ TEST_CASE( "MultipathMapper can map to a one-node graph", "[multipath][mapping][ })"; // Load the JSON + Graph proto_graph; + json2pb(proto_graph, graph_json.c_str(), graph_json.size()); + // Make it into a VG - auto graph = json_to_graph(graph_json); + VG graph; + graph.extend(proto_graph); // Make GCSA quiet gcsa::Verbosity::set(gcsa::Verbosity::SILENT); @@ -283,11 +290,11 @@ TEST_CASE( "MultipathMapper can map to a one-node graph", "[multipath][mapping][ gcsa::LCPArray* lcpidx = nullptr; // Build the GCSA index - build_gcsa_lcp(*graph, gcsaidx, lcpidx, 16, 3); + build_gcsa_lcp(graph, gcsaidx, lcpidx, 16, 3); // Build the xg index xg::XG xg_index; - xg_index.from_path_handle_graph(*graph); + xg_index.from_path_handle_graph(graph); // Make a multipath mapper to map against the graph. MultipathMapper mapper(&xg_index, gcsaidx, lcpidx); @@ -419,8 +426,12 @@ TEST_CASE( "MultipathMapper can work on a bigger graph", "[multipath][mapping][m string graph_json = R"({"node":[{"sequence":"CTTCTCATCCCTCCTCAAGGGCCTTTAACTACTCCACATCCAAAGCTACCCAGGCCATTTTAAGTTTCCTGTGGACTAAGGACAAAGGTGCGGGGAGATG","id":12},{"sequence":"A","id":2},{"sequence":"CAAATAAGGCTTGGAAATTTTCTGGAGTTCTATTATATTCCAACTCTCTGGTTCCTGGTGCTATGTGTAACTAGTAATGGTAATGGATATGTTGGGCTTT","id":3},{"sequence":"TTTCTTTGATTTATTTGAAGTGACGTTTGACAATCTATCACTAGGGGTAATGTGGGGAAATGGAAAGAATACAAGATTTGGAGCCAGACAAATCTGGGTT","id":4},{"sequence":"CAAATCCTCACTTTGCCACATATTAGCCATGTGACTTTGAACAAGTTAGTTAATCTCTCTGAACTTCAGTTTAATTATCTCTAATATGGAGATGATACTA","id":5},{"sequence":"CTGACAGCAGAGGTTTGCTGTGAAGATTAAATTAGGTGATGCTTGTAAAGCTCAGGGAATAGTGCCTGGCATAGAGGAAAGCCTCTGACAACTGGTAGTT","id":6},{"sequence":"ACTGTTATTTACTATGAATCCTCACCTTCCTTGACTTCTTGAAACATTTGGCTATTGACCTCTTTCCTCCTTGAGGCTCTTCTGGCTTTTCATTGTCAAC","id":7},{"sequence":"ACAGTCAACGCTCAATACAAGGGACATTAGGATTGGCAGTAGCTCAGAGATCTCTCTGCTCACCGTGATCTTCAAGTTTGAAAATTGCATCTCAAATCTA","id":8},{"sequence":"AGACCCAGAGGGCTCACCCAGAGTCGAGGCTCAAGGACAGCTCTCCTTTGTGTCCAGAGTGTATACGATGTAACTCTGTTCGGGCACTGGTGAAAGATAA","id":9},{"sequence":"CAGAGGAAATGCCTGGCTTTTTATCAGAACATGTTTCCAAGCTTATCCCTTTTCCCAGCTCTCCTTGTCCCTCCCAAGATCTCTTCACTGGCCTCTTATC","id":10},{"sequence":"TTTACTGTTACCAAATCTTTCCAGAAGCTGCTCTTTCCCTCAATTGTTCATTTGTCTTCTTGTCCAGGAATGAACCACTGCTCTCTTCTTGTCAGATCAG","id":11}],"path":[{"name":"x","mapping":[{"position":{"node_id":3},"edit":[{"from_length":100,"to_length":100}],"rank":1},{"position":{"node_id":4},"edit":[{"from_length":100,"to_length":100}],"rank":2},{"position":{"node_id":5},"edit":[{"from_length":100,"to_length":100}],"rank":3},{"position":{"node_id":6},"edit":[{"from_length":100,"to_length":100}],"rank":4},{"position":{"node_id":7},"edit":[{"from_length":100,"to_length":100}],"rank":5},{"position":{"node_id":8},"edit":[{"from_length":100,"to_length":100}],"rank":6},{"position":{"node_id":9},"edit":[{"from_length":100,"to_length":100}],"rank":7},{"position":{"node_id":10},"edit":[{"from_length":100,"to_length":100}],"rank":8},{"position":{"node_id":11},"edit":[{"from_length":100,"to_length":100}],"rank":9},{"position":{"node_id":12},"edit":[{"from_length":100,"to_length":100}],"rank":10},{"position":{"node_id":2},"edit":[{"from_length":1,"to_length":1}],"rank":11}]}],"edge":[{"from":12,"to":2},{"from":3,"to":4},{"from":4,"to":5},{"from":5,"to":6},{"from":6,"to":7},{"from":7,"to":8},{"from":8,"to":9},{"from":9,"to":10},{"from":10,"to":11},{"from":11,"to":12}]})"; // Load the JSON + Graph proto_graph; + json2pb(proto_graph, graph_json.c_str(), graph_json.size()); + // Make it into a VG - auto graph = json_to_graph(graph_json); + VG graph; + graph.extend(proto_graph); // Make GCSA quiet gcsa::Verbosity::set(gcsa::Verbosity::SILENT); @@ -430,7 +441,7 @@ TEST_CASE( "MultipathMapper can work on a bigger graph", "[multipath][mapping][m gcsa::LCPArray* lcpidx = nullptr; // Build the GCSA index - build_gcsa_lcp(*graph, gcsaidx, lcpidx, 16, 3); + build_gcsa_lcp(graph, gcsaidx, lcpidx, 16, 3); // Build the xg index xg::XG xg_index; diff --git a/src/unittest/path_component_index.cpp b/src/unittest/path_component_index.cpp index bfdda70d64..058f4bf9c1 100644 --- a/src/unittest/path_component_index.cpp +++ b/src/unittest/path_component_index.cpp @@ -9,7 +9,6 @@ #include "xg.hpp" #include "vg.hpp" #include "vg/io/json2pb.h" -#include "support/json.hpp" #include namespace vg { @@ -19,9 +18,13 @@ namespace unittest { string graph_json = R"({"node": [{"sequence": "AAACCC", "id": 1}, {"sequence": "CACACA", "id": 2}, {"sequence": "CACACA", "id": 3}, {"sequence": "TTTTGG", "id": 4}, {"sequence": "ACGTAC", "id": 5}], "path": [{"name": "one", "mapping": [{"position": {"node_id": 1}, "rank": 1}, {"position": {"node_id": 2}, "rank": 2}]}, {"name": "three", "mapping": [{"position": {"node_id": 2}, "rank": 1}, {"position": {"node_id": 3}, "rank": 2}]}, {"name": "two", "mapping": [{"position": {"node_id": 4}, "rank": 1}, {"position": {"node_id": 5}, "rank": 2}]}], "edge": [{"from": 1, "to": 2}, {"from": 2, "to": 3}, {"from": 4, "to": 5}]})"; - // Load the JSON and build the xg index + // Load the JSON + Graph proto_graph; + json2pb(proto_graph, graph_json.c_str(), graph_json.size()); + + // Build the xg index xg::XG xg_index; - xg_index.from_path_handle_graph(*json_to_graph(graph_json)); + xg_index.from_path_handle_graph(VG(proto_graph)); unordered_set comp_1; diff --git a/src/unittest/phase_unfolder.cpp b/src/unittest/phase_unfolder.cpp index 67c079ed19..0c79972941 100644 --- a/src/unittest/phase_unfolder.cpp +++ b/src/unittest/phase_unfolder.cpp @@ -14,7 +14,6 @@ #include "../phase_unfolder.hpp" #include "vg/io/json2pb.h" #include "xg.hpp" -#include "support/json.hpp" #include "catch.hpp" @@ -211,8 +210,10 @@ const std::string unfolder_graph_path = R"( TEST_CASE("PhaseUnfolder can unfold XG paths", "[phaseunfolder][indexing]") { // Build an XG index with a path. + Graph graph_with_path; + json2pb(graph_with_path, unfolder_graph_path.c_str(), unfolder_graph_path.size()); xg::XG xg_index; - xg_index.from_path_handle_graph(*json_to_graph(unfolder_graph_path)); + xg_index.from_path_handle_graph(VG(graph_with_path)); // Build an empty GBWT index. gbwt::GBWT gbwt_index; @@ -222,7 +223,10 @@ TEST_CASE("PhaseUnfolder can unfold XG paths", "[phaseunfolder][indexing]") { PhaseUnfolder unfolder(xg_index, gbwt_index, next_id); // Build a VG graph. - auto vg_graph = json_to_graph(unfolder_graph); + VG vg_graph; + Graph temp_graph; + json2pb(temp_graph, unfolder_graph.c_str(), unfolder_graph.size()); + vg_graph.merge(temp_graph); // Remove branching regions from the VG graph, including the last node, // but keep the edge (1, 6) in the graph. @@ -251,8 +255,10 @@ TEST_CASE("PhaseUnfolder can unfold XG paths", "[phaseunfolder][indexing]") { TEST_CASE("PhaseUnfolder can restore XG paths", "[phaseunfolder][indexing]") { // Build an XG index with a path. + Graph graph_with_path; + json2pb(graph_with_path, unfolder_graph_path.c_str(), unfolder_graph_path.size()); xg::XG xg_index; - xg_index.from_path_handle_graph(*json_to_graph(unfolder_graph_path)); + xg_index.from_path_handle_graph(VG(graph_with_path)); // Build an empty GBWT index. gbwt::GBWT gbwt_index; @@ -262,7 +268,10 @@ TEST_CASE("PhaseUnfolder can restore XG paths", "[phaseunfolder][indexing]") { PhaseUnfolder unfolder(xg_index, gbwt_index, next_id); // Build a VG graph. - auto vg_graph = json_to_graph(unfolder_graph); + VG vg_graph; + Graph temp_graph; + json2pb(temp_graph, unfolder_graph.c_str(), unfolder_graph.size()); + vg_graph.merge(temp_graph); // Remove branching regions from the VG graph, including the last node, // but keep the edge (1, 6) in the graph. @@ -325,7 +334,10 @@ TEST_CASE("PhaseUnfolder can unfold GBWT threads", "[phaseunfolder][indexing]") PhaseUnfolder unfolder(xg_index, gbwt_index, next_id); // Build a VG graph. - auto vg_graph = json_to_graph(unfolder_graph); + VG vg_graph; + Graph temp_graph; + json2pb(temp_graph, unfolder_graph.c_str(), unfolder_graph.size()); + vg_graph.merge(temp_graph); // Remove branching regions from the VG graph, including the last node, // but keep the edge (1, 6) in the graph. @@ -354,8 +366,10 @@ TEST_CASE("PhaseUnfolder can unfold GBWT threads", "[phaseunfolder][indexing]") TEST_CASE("PhaseUnfolder can unfold both XG paths and GBWT threads", "[phaseunfolder][indexing]") { // Build an XG index with a path. + Graph graph_with_path; + json2pb(graph_with_path, unfolder_graph_path.c_str(), unfolder_graph_path.size()); xg::XG xg_index; - xg_index.from_path_handle_graph(*json_to_graph(unfolder_graph_path)); + xg_index.from_path_handle_graph(VG(graph_with_path)); // Build a GBWT with three threads including a duplicate. We want to have // only one instance of short_path unfolded, but we want separate copies @@ -387,7 +401,10 @@ TEST_CASE("PhaseUnfolder can unfold both XG paths and GBWT threads", "[phaseunfo PhaseUnfolder unfolder(xg_index, gbwt_index, next_id); // Build a VG graph. - auto vg_graph = json_to_graph(unfolder_graph); + VG vg_graph; + Graph temp_graph; + json2pb(temp_graph, unfolder_graph.c_str(), unfolder_graph.size()); + vg_graph.merge(temp_graph); // Remove branching regions from the VG graph, including the last node, // but keep the edge (1, 6) in the graph. diff --git a/src/unittest/readfilter.cpp b/src/unittest/readfilter.cpp index 49ef3bb389..cc1562f3f3 100644 --- a/src/unittest/readfilter.cpp +++ b/src/unittest/readfilter.cpp @@ -5,7 +5,6 @@ #include "catch.hpp" #include "readfilter.hpp" #include "xg.hpp" -#include "support/json.hpp" namespace vg { namespace unittest { @@ -46,9 +45,12 @@ TEST_CASE("reads with ambiguous ends can be trimmed", "[filter]") { )"; // Load it into Protobuf + Graph chunk; + json2pb(chunk, graph_json.c_str(), graph_json.size()); + // Pass it over to XG xg::XG index; - index.from_path_handle_graph(*json_to_graph(graph_json)); + index.from_path_handle_graph(VG(chunk)); // Make a ReadFilter; ReadFilter filter; diff --git a/src/unittest/snarl_distance_index.cpp b/src/unittest/snarl_distance_index.cpp index 947117dbd4..093c45d0aa 100644 --- a/src/unittest/snarl_distance_index.cpp +++ b/src/unittest/snarl_distance_index.cpp @@ -14,7 +14,6 @@ #include "catch.hpp" #include "support/random_graph.hpp" #include "support/randomness.hpp" -#include "support/json.hpp" #include "../snarl_distance_index.hpp" #include "../integrated_snarl_finder.hpp" #include "../genotypekit.hpp" @@ -3755,7 +3754,9 @@ namespace vg { // Make an actual graph VG graph; - auto chunk_ptr = json_to_graph(graph_json); + Graph chunk; + json2pb(chunk, graph_json.c_str(), graph_json.size()); + graph.extend(chunk); // We need to see the path. REQUIRE(graph.paths.size() == 1); @@ -4013,7 +4014,9 @@ namespace vg { // Make an actual graph VG graph; - auto chunk_ptr = json_to_graph(graph_json); + Graph chunk; + json2pb(chunk, graph_json.c_str(), graph_json.size()); + graph.extend(chunk); IntegratedSnarlFinder snarl_finder(graph); SnarlDistanceIndex distance_index; @@ -4124,7 +4127,9 @@ namespace vg { // Make an actual graph VG graph; - auto chunk_ptr = json_to_graph(graph_json); + Graph chunk; + json2pb(chunk, graph_json.c_str(), graph_json.size()); + graph.extend(chunk); IntegratedSnarlFinder snarl_finder(graph); SnarlDistanceIndex distance_index; @@ -4271,7 +4276,9 @@ namespace vg { // Make an actual graph VG graph; - auto chunk_ptr = json_to_graph(graph_json); + Graph chunk; + json2pb(chunk, graph_json.c_str(), graph_json.size()); + graph.extend(chunk); IntegratedSnarlFinder snarl_finder(graph); SnarlDistanceIndex distance_index; @@ -4398,7 +4405,9 @@ namespace vg { // Make an actual graph VG graph; - auto chunk_ptr = json_to_graph(graph_json); + Graph chunk; + json2pb(chunk, graph_json.c_str(), graph_json.size()); + graph.extend(chunk); IntegratedSnarlFinder snarl_finder(graph); SnarlDistanceIndex distance_index; @@ -4505,7 +4514,9 @@ namespace vg { // Make an actual graph VG graph; - auto chunk_ptr = json_to_graph(graph_json); + Graph chunk; + json2pb(chunk, graph_json.c_str(), graph_json.size()); + graph.extend(chunk); IntegratedSnarlFinder snarl_finder(graph); SnarlDistanceIndex distance_index; @@ -4607,7 +4618,9 @@ namespace vg { // Make an actual graph VG graph; - auto chunk_ptr = json_to_graph(graph_json); + Graph chunk; + json2pb(chunk, graph_json.c_str(), graph_json.size()); + graph.extend(chunk); IntegratedSnarlFinder snarl_finder(graph); SnarlDistanceIndex distance_index; @@ -4775,7 +4788,9 @@ namespace vg { // Make an actual graph VG graph; - auto chunk_ptr = json_to_graph(graph_json); + Graph chunk; + json2pb(chunk, graph_json.c_str(), graph_json.size()); + graph.extend(chunk); IntegratedSnarlFinder snarl_finder(graph); SnarlDistanceIndex distance_index; @@ -4896,7 +4911,9 @@ namespace vg { // Make an actual graph VG graph; - auto chunk_ptr = json_to_graph(graph_json); + Graph chunk; + json2pb(chunk, graph_json.c_str(), graph_json.size()); + graph.extend(chunk); IntegratedSnarlFinder snarl_finder(graph); SnarlDistanceIndex distance_index; diff --git a/src/unittest/snarls.cpp b/src/unittest/snarls.cpp index 4d6be27383..c2f5030326 100644 --- a/src/unittest/snarls.cpp +++ b/src/unittest/snarls.cpp @@ -13,7 +13,6 @@ #include "catch.hpp" #include "support/random_graph.hpp" #include "support/randomness.hpp" -#include "support/json.hpp" #include "../snarls.hpp" #include "../cactus_snarl_finder.hpp" #include "../integrated_snarl_finder.hpp" @@ -1702,7 +1701,9 @@ namespace vg { VG graph; // Load up the graph - auto g_ptr = json_to_graph(graph_json); + Graph g; + json2pb(g, graph_json.c_str(), graph_json.size()); + graph.extend(g); // Define the one snarl Snarl snarl1; @@ -1833,7 +1834,9 @@ namespace vg { VG graph; // Load up the graph - auto g_ptr = json_to_graph(graph_json); + Graph g; + json2pb(g, graph_json.c_str(), graph_json.size()); + graph.extend(g); // Load the snarls Snarl snarl1, snarl2, snarl3; @@ -1917,7 +1920,9 @@ namespace vg { // Make an actual graph VG graph; - auto chunk_ptr = json_to_graph(graph_json); + Graph chunk; + json2pb(chunk, graph_json.c_str(), graph_json.size()); + graph.extend(chunk); // We need to see the path. REQUIRE(graph.paths.size() == 1); @@ -2040,7 +2045,9 @@ namespace vg { // Make an actual graph VG graph; - auto chunk_ptr = json_to_graph(graph_json); + Graph chunk; + json2pb(chunk, graph_json.c_str(), graph_json.size()); + graph.extend(chunk); SnarlManager snarl_manager = CactusSnarlFinder(graph).find_snarls(); @@ -2125,7 +2132,9 @@ namespace vg { // Make an actual graph VG graph; - auto chunk_ptr = json_to_graph(graph_json); + Graph chunk; + json2pb(chunk, graph_json.c_str(), graph_json.size()); + graph.extend(chunk); SnarlManager snarl_manager = CactusSnarlFinder(graph).find_snarls(); @@ -2242,7 +2251,9 @@ namespace vg { // Make an actual graph VG graph; - auto chunk_ptr = json_to_graph(graph_json); + Graph chunk; + json2pb(chunk, graph_json.c_str(), graph_json.size()); + graph.extend(chunk); SnarlManager snarl_manager = CactusSnarlFinder(graph).find_snarls(); @@ -2351,7 +2362,9 @@ namespace vg { // Make an actual graph VG graph; - auto chunk_ptr = json_to_graph(graph_json); + Graph chunk; + json2pb(chunk, graph_json.c_str(), graph_json.size()); + graph.extend(chunk); SnarlManager snarl_manager = CactusSnarlFinder(graph).find_snarls(); @@ -2407,7 +2420,9 @@ namespace vg { // Make an actual graph VG graph; - auto chunk_ptr = json_to_graph(graph_json); + Graph chunk; + json2pb(chunk, graph_json.c_str(), graph_json.size()); + graph.extend(chunk); SnarlManager snarl_manager = CactusSnarlFinder(graph).find_snarls(); @@ -2483,7 +2498,9 @@ namespace vg { // Make an actual graph VG graph; - auto chunk_ptr = json_to_graph(graph_json); + Graph chunk; + json2pb(chunk, graph_json.c_str(), graph_json.size()); + graph.extend(chunk); SnarlManager snarl_manager = CactusSnarlFinder(graph).find_snarls(); @@ -2543,7 +2560,9 @@ namespace vg { // Make an actual graph VG graph; - auto chunk_ptr = json_to_graph(graph_json); + Graph chunk; + json2pb(chunk, graph_json.c_str(), graph_json.size()); + graph.extend(chunk); SnarlManager snarl_manager = CactusSnarlFinder(graph).find_snarls(); @@ -2753,7 +2772,9 @@ namespace vg { // Make an actual graph VG graph; - auto chunk_ptr = json_to_graph(graph_json); + Graph chunk; + json2pb(chunk, graph_json.c_str(), graph_json.size()); + graph.extend(chunk); SnarlManager snarl_manager = CactusSnarlFinder(graph).find_snarls(); #ifdef debug @@ -3903,7 +3924,9 @@ namespace vg { // Make an actual graph VG graph; - auto chunk_ptr = json_to_graph(graph_json); + Graph chunk; + json2pb(chunk, graph_json.c_str(), graph_json.size()); + graph.extend(chunk); assert(graph.is_valid()); SECTION( "PathTraversalFinder can find simple forward traversals") { diff --git a/src/unittest/variant_adder.cpp b/src/unittest/variant_adder.cpp index 69d861ebbf..afe3353e4b 100644 --- a/src/unittest/variant_adder.cpp +++ b/src/unittest/variant_adder.cpp @@ -10,7 +10,6 @@ #include "../utility.hpp" #include "../path.hpp" #include "vg/io/json2pb.h" -#include "support/json.hpp" #include #include @@ -53,11 +52,15 @@ ref 5 rs1337 A G 29 PASS . GT ] })"; - // Load the JSON and make it into a VG - auto graph_ptr = json_to_graph(graph_json); - VG& graph = *dynamic_cast(graph_ptr.get()); - - + // Load the JSON + Graph proto_graph; + json2pb(proto_graph, graph_json.c_str(), graph_json.size()); + + // Make it into a VG + VG graph; + graph.extend(proto_graph); + + // Make a VariantAdder VariantAdder adder(graph); // Fail to add the variants to the graph @@ -96,10 +99,14 @@ ref 5 rs1337 A G 29 PASS . GT 0/1 ] })"; - // Load the JSON and make it into a VG - auto graph_ptr = json_to_graph(graph_json); - VG& graph = *dynamic_cast(graph_ptr.get()); - + // Load the JSON + Graph proto_graph; + json2pb(proto_graph, graph_json.c_str(), graph_json.size()); + + // Make it into a VG + VG graph; + graph.extend(proto_graph); + // Make a VariantAdder VariantAdder adder(graph); // Add the variants to the graph @@ -146,10 +153,14 @@ ref 5 rs1337 AAAAAAAAAAAAAAAAAAAAA A 29 PASS . GT 0/1 ] })"; - // Load the JSON and make it into a VG - auto graph_ptr = json_to_graph(graph_json); - VG& graph = *dynamic_cast(graph_ptr.get()); - + // Load the JSON + Graph proto_graph; + json2pb(proto_graph, graph_json.c_str(), graph_json.size()); + + // Make it into a VG + VG graph; + graph.extend(proto_graph); + // Make a VariantAdder VariantAdder adder(graph); // Add the variants to the graph @@ -203,12 +214,16 @@ ref 5 rs1337 AAAAAAAAAAAAAAAAAAAAA A 29 PASS . GT 0/1 ] })"; - // Load the JSON and make it into a VG - auto graph_ptr = json_to_graph(graph_json); - VG& graph = *dynamic_cast(graph_ptr.get()); - + // Load the JSON + Graph proto_graph; + json2pb(proto_graph, graph_json.c_str(), graph_json.size()); + + // Make it into a VG + VG graph; + graph.extend(proto_graph); + SECTION ("should work when the graph is as given") { - + // Make a VariantAdder VariantAdder adder(graph); // Add the variants to the graph @@ -279,10 +294,14 @@ ref 5 rs1337 AAAAAAAAAAAAAAAAAAAAA AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA 29 ] })"; - // Load the JSON and make it into a VG - auto graph_ptr = json_to_graph(graph_json); - VG& graph = *dynamic_cast(graph_ptr.get()); - + // Load the JSON + Graph proto_graph; + json2pb(proto_graph, graph_json.c_str(), graph_json.size()); + + // Make it into a VG + VG graph; + graph.extend(proto_graph); + // Make a VariantAdder VariantAdder adder(graph); adder.skip_structural_duplications = true; @@ -305,10 +324,14 @@ TEST_CASE( "The smart aligner works on very large inserts", "[variantadder]" ) { "node": [{"id": 1, "sequence": "GCGCAAAAAAAAAAAAAAAAAAAAAGCGC"}] })"; - // Load the JSON and make it into a VG - auto graph_ptr = json_to_graph(graph_json); - VG& graph = *dynamic_cast(graph_ptr.get()); - + // Load the JSON + Graph proto_graph; + json2pb(proto_graph, graph_json.c_str(), graph_json.size()); + + // Make it into a VG + VG graph; + graph.extend(proto_graph); + // Make a VariantAdder VariantAdder adder(graph); vector order = handlealgs::topological_order(&adder.get_graph()); @@ -381,10 +404,14 @@ TEST_CASE( "The smart aligner should use mapping offsets on huge deletions", "[v } graph_json = regex_replace(graph_json, std::regex("<10kAs>"), a_stream.str()); - // Load the JSON and make it into a VG - auto graph_ptr = json_to_graph(graph_json); - VG& graph = *dynamic_cast(graph_ptr.get()); - + // Load the JSON + Graph proto_graph; + json2pb(proto_graph, graph_json.c_str(), graph_json.size()); + + // Make it into a VG + VG graph; + graph.extend(proto_graph); + // Make a VariantAdder VariantAdder adder(graph); vector order = handlealgs::topological_order(&adder.get_graph()); @@ -465,10 +492,14 @@ TEST_CASE( "The smart aligner should find existing huge deletions", "[variantadd } graph_json = regex_replace(graph_json, std::regex("<10kAs>"), a_stream.str()); - // Load the JSON and make it into a VG - auto graph_ptr = json_to_graph(graph_json); - VG& graph = *dynamic_cast(graph_ptr.get()); - + // Load the JSON + Graph proto_graph; + json2pb(proto_graph, graph_json.c_str(), graph_json.size()); + + // Make it into a VG + VG graph; + graph.extend(proto_graph); + // Make a VariantAdder VariantAdder adder(graph); vector order = handlealgs::topological_order(&adder.get_graph()); @@ -541,13 +572,17 @@ TEST_CASE( "The smart aligner should use deletion edits on medium deletions", "[ } graph_json = regex_replace(graph_json, std::regex("<100As>"), a_stream.str()); - // Load the JSON and make it into a VG - auto graph_ptr = json_to_graph(graph_json); - VG& graph = *dynamic_cast(graph_ptr.get()); - + // Load the JSON + Graph proto_graph; + json2pb(proto_graph, graph_json.c_str(), graph_json.size()); + + // Make it into a VG + VG graph; + graph.extend(proto_graph); + // Make a VariantAdder VariantAdder adder(graph); - + // Make a deleted version (only 21 As) string deleted = "GCGCAAAAAAAAAAAAAAAAAAAAAGCGC"; diff --git a/src/unittest/vg.cpp b/src/unittest/vg.cpp index 81b3b96424..9beb3e1ca7 100644 --- a/src/unittest/vg.cpp +++ b/src/unittest/vg.cpp @@ -9,7 +9,6 @@ #include "../algorithms/normalize.hpp" #include "../algorithms/disjoint_components.hpp" #include "handle.hpp" -#include "support/json.hpp" namespace vg { namespace unittest { @@ -18,8 +17,10 @@ using namespace std; // Turn a JSON string into a VG graph VG string_to_graph(const string& json) { - auto graph_ptr = json_to_graph(json); - VG& graph = *dynamic_cast(graph_ptr.get()); + VG graph; + Graph chunk; + json2pb(chunk, json.c_str(), json.size()); + graph.merge(chunk); return graph; } diff --git a/src/unittest/vg_algorithms.cpp b/src/unittest/vg_algorithms.cpp index 380946e5eb..b4fc736734 100644 --- a/src/unittest/vg_algorithms.cpp +++ b/src/unittest/vg_algorithms.cpp @@ -28,7 +28,6 @@ #include "../xg.hpp" #include #include "vg/io/json2pb.h" -#include "support/json.hpp" using namespace google::protobuf; @@ -1093,7 +1092,11 @@ TEST_CASE( "Connecting graph extraction works on a cool loop without leaving ext {"edge": [{"from": "185927720", "to": "185927722"}, {"from": "185927721", "from_start": true, "to": "185927722"}, {"from": "185927722", "to": "186681786", "to_end": true}, {"from": "185927722", "to": "185927723"}, {"from": "186681786", "to": "186683083"}, {"from": "186681786", "from_start": true, "to": "186681787", "to_end": true}, {"from": "186681787", "to": "186683069", "to_end": true}, {"from": "186681787", "from_start": true, "to": "186681789"}, {"from": "186681787", "from_start": true, "to": "186681788", "to_end": true}, {"from": "186681788", "from_start": true, "to": "186681790", "to_end": true}, {"from": "186681789", "to": "186681790", "to_end": true}, {"from": "186681790", "from_start": true, "to": "186681792", "to_end": true}, {"from": "186683069", "from_start": true, "to": "186683079", "to_end": true}, {"from": "186683079", "from_start": true, "to": "186683080", "to_end": true}, {"from": "186683080", "from_start": true, "to": "186683081", "to_end": true}, {"from": "186683081", "from_start": true, "to": "186683083", "to_end": true}], "node": [{"id": "185927720", "sequence": "G"}, {"id": "185927721", "sequence": "A"}, {"id": "185927722", "sequence": "ACCGGG"}, {"id": "185927723", "sequence": "AGTGGGGG"}, {"id": "186681786", "sequence": "C"}, {"id": "186681787", "sequence": "TGGGAGTCTAAGTCTCTTTTGATCACACTTTAAAGACCAAAAGGTAGAAGCGCAAAGACGTTATCTGTCCAATATTACAAACCTAGTAAGTGGTGGAATTTGGCCTTGAACCCAGATCTGTAACTCCAGAGCCGAAGTGCTTCACCCACCTCCCTGTGGTG"}, {"id": "186681788", "sequence": "G"}, {"id": "186681789", "sequence": "T"}, {"id": "186681790", "sequence": "TAT"}, {"id": "186681792", "sequence": "T"}, {"id": "186683069", "sequence": "G"}, {"id": "186683079", "sequence": "G"}, {"id": "186683080", "sequence": "TACCCCGGAATCCCTGCCGCGGCCCCTCGGGCCTGTCCACATCCCTCTGCCCCTCCCAGACCTCTGTCCTTCCACCAATCGCCTCCCGCAGCCCCGAGCCGCCACTCCCAGTCCCCCGAGTCCCTGCCGCGCGCCCTCGCGCCTGTCCACATCCCTCTGCCCATCCGAGACCTCTGTCCTTACACCACTAGCCACCCCACGTGGGACTTCCATGGCTTCTGAGTACAAGGCCAGCCCCCCGGCCCACCAGCTTTCGGAATGCCTGCTTACCTCTTTTTCTGTAGA"}, {"id": "186683081", "sequence": "CCGG"}, {"id": "186683083", "sequence": "C"}]} )"; - auto source_graph = json_to_graph(graph_json); + Graph source; + json2pb(source, graph_json.c_str(), graph_json.size()); + + VG vg; + vg.extend(source); bdsg::HashGraph extractor; @@ -1102,7 +1105,7 @@ TEST_CASE( "Connecting graph extraction works on a cool loop without leaving ext pos_t dest_pos = make_pos_t(186681787, true, 131); // If we have strict max length set to false, we may get extra tips. - unordered_map connect_trans = algorithms::extract_connecting_graph(source_graph.get(), &extractor, max_dist, src_pos, dest_pos, false); + unordered_map connect_trans = algorithms::extract_connecting_graph(&vg, &extractor, max_dist, src_pos, dest_pos, false); std::vector tip_handles = handlegraph::algorithms::find_tips(&extractor); // There ought to be at least the two tips REQUIRE(tip_handles.size() >= 2); @@ -1110,7 +1113,7 @@ TEST_CASE( "Connecting graph extraction works on a cool loop without leaving ext extractor.clear(); // If we have strict connecting-ness set to true, we won't get any extra tips. - connect_trans = algorithms::extract_connecting_graph(source_graph.get(), &extractor, max_dist, src_pos, dest_pos, true); + connect_trans = algorithms::extract_connecting_graph(&vg, &extractor, max_dist, src_pos, dest_pos, true); tip_handles = handlegraph::algorithms::find_tips(&extractor); // There ought to be just the two tips REQUIRE(tip_handles.size() == 2); @@ -1685,7 +1688,11 @@ TEST_CASE( "Connecting graph extraction works on a particular case without leavi )"; - auto vg = json_to_graph(graph_json); + Graph source; + json2pb(source, graph_json.c_str(), graph_json.size()); + + VG vg; + vg.extend(source); VG extractor; @@ -5378,7 +5385,11 @@ TEST_CASE("simplify_siblings() works on a graph with a reversing self loop", "[a {"edge": [{"from": "1", "to": "3"}, {"from": "1", "to": "2"}, {"from": "2", "to": "2", "to_end": true}], "node": [{"id": "1", "sequence": "T"}, {"id": "2", "sequence": "A"}, {"id": "3", "sequence": "ACA"}], "path": [{"mapping": [{"edit": [{"from_length": 1, "to_length": 1}], "position": {"node_id": "1"}, "rank": "1"}, {"edit": [{"from_length": 1, "to_length": 1}], "position": {"node_id": "2"}, "rank": "2"}, {"edit": [{"from_length": 1, "to_length": 1}], "position": {"is_reverse": true, "node_id": "2"}, "rank": "3"}], "name": "x"}, {"mapping": [{"edit": [{"from_length": 1, "to_length": 1}], "position": {"node_id": "1"}, "rank": "1"}, {"edit": [{"from_length": 3, "to_length": 3}], "position": {"node_id": "3"}, "rank": "2"}], "name": "y"}]} )"; - auto graph = json_to_graph(graph_json); + Graph source; + json2pb(source, graph_json.c_str(), graph_json.size()); + + VG graph; + graph.extend(source); @@ -5394,7 +5405,11 @@ TEST_CASE("simplify_siblings() works on a smaller graph with a reversing self lo {"edge": [{"from": "1", "to": "3"}, {"from": "1", "to": "2"}, {"from": "2", "to": "2", "to_end": true}], "node": [{"id": "1", "sequence": "T"}, {"id": "2", "sequence": "A"}, {"id": "3", "sequence": "A"}], "path": [{"mapping": [{"edit": [{"from_length": 1, "to_length": 1}], "position": {"node_id": "1"}, "rank": "1"}, {"edit": [{"from_length": 1, "to_length": 1}], "position": {"node_id": "2"}, "rank": "2"}, {"edit": [{"from_length": 1, "to_length": 1}], "position": {"is_reverse": true, "node_id": "2"}, "rank": "3"}], "name": "x"}]} )"; - auto graph = json_to_graph(graph_json); + Graph source; + json2pb(source, graph_json.c_str(), graph_json.size()); + + VG graph; + graph.extend(source); @@ -5410,7 +5425,11 @@ TEST_CASE("normalize() works on a graph with a reversing self loop", "[algorithm {"edge": [{"from": "1", "to": "3"}, {"from": "1", "to": "2"}, {"from": "2", "to": "2", "to_end": true}], "node": [{"id": "1", "sequence": "T"}, {"id": "2", "sequence": "A"}, {"id": "3", "sequence": "ACA"}], "path": [{"mapping": [{"edit": [{"from_length": 1, "to_length": 1}], "position": {"node_id": "1"}, "rank": "1"}, {"edit": [{"from_length": 1, "to_length": 1}], "position": {"node_id": "2"}, "rank": "2"}, {"edit": [{"from_length": 1, "to_length": 1}], "position": {"is_reverse": true, "node_id": "2"}, "rank": "3"}], "name": "x"}, {"mapping": [{"edit": [{"from_length": 1, "to_length": 1}], "position": {"node_id": "1"}, "rank": "1"}, {"edit": [{"from_length": 3, "to_length": 3}], "position": {"node_id": "3"}, "rank": "2"}], "name": "y"}]} )"; - auto graph = json_to_graph(graph_json); + Graph source; + json2pb(source, graph_json.c_str(), graph_json.size()); + + VG graph; + graph.extend(source); diff --git a/src/unittest/vpkg.cpp b/src/unittest/vpkg.cpp index 6e1ed8e4ab..51a849c446 100644 --- a/src/unittest/vpkg.cpp +++ b/src/unittest/vpkg.cpp @@ -14,7 +14,6 @@ #include "../vg.hpp" #include "../snarl_seed_clusterer.hpp" #include "vg/io/json2pb.h" -#include "support/json.hpp" #include #include #include @@ -50,9 +49,13 @@ TEST_CASE("We can read and write XG", "[vpkg][handlegraph][xg]") { "edge":[{"to":2,"from":1}]} )"; - // Load the JSON and build the xg index + // Load the JSON + Graph proto_graph; + json2pb(proto_graph, graph_json.c_str(), graph_json.size()); + + // Build the xg index xg::XG xg_index; - xg_index.from_path_handle_graph(*json_to_graph(graph_json)); + xg_index.from_path_handle_graph(VG(proto_graph)); stringstream ss; @@ -146,8 +149,12 @@ TEST_CASE("We can read VG from a VPKG-wrapped stream as a VG", "[vpkg][handlegra "edge":[{"to":2,"from":1}]} )"; - // Load the JSON and build the VG - auto vg_graph = json_to_graph(graph_json); + // Load the JSON + Graph proto_graph; + json2pb(proto_graph, graph_json.c_str(), graph_json.size()); + + // Build the VG + vg::VG vg_graph(proto_graph); // Save it stringstream ss; @@ -173,8 +180,12 @@ TEST_CASE("We can read VG from a VPKG-wrapped stream as a HandleGraph which is a "edge":[{"to":2,"from":1}]} )"; - // Load the JSON and build the VG - auto vg_graph = json_to_graph(graph_json); + // Load the JSON + Graph proto_graph; + json2pb(proto_graph, graph_json.c_str(), graph_json.size()); + + // Build the VG + vg::VG vg_graph(proto_graph); // Save it stringstream ss; @@ -200,8 +211,12 @@ TEST_CASE("We can read VG from a VPKG-wrapped stream as a HandleGraph which is a TEST_CASE("We can read an empty VG as a HandleGraph", "[vpkg][handlegraph][vg][empty]") { string graph_json = "{}"; - // Load the JSON and build the VG - auto vg_graph = json_to_graph(graph_json); + // Load the JSON + Graph proto_graph; + json2pb(proto_graph, graph_json.c_str(), graph_json.size()); + + // Build the VG + vg::VG vg_graph(proto_graph); // Save it stringstream ss; @@ -226,8 +241,12 @@ TEST_CASE("We prefer to read a graph as the first provided type that matches", " "edge":[{"to":2,"from":1}]} )"; - // Load the JSON and build the VG - auto vg_graph = json_to_graph(graph_json); + // Load the JSON + Graph proto_graph; + json2pb(proto_graph, graph_json.c_str(), graph_json.size()); + + // Build the VG + vg::VG vg_graph(proto_graph); // Save it stringstream ss; diff --git a/src/unittest/xdrop_aligner.cpp b/src/unittest/xdrop_aligner.cpp index f943b0bf3c..f745b8f66a 100644 --- a/src/unittest/xdrop_aligner.cpp +++ b/src/unittest/xdrop_aligner.cpp @@ -11,7 +11,6 @@ #include #include "test_aligner.hpp" #include "catch.hpp" -#include "support/json.hpp" #include "bdsg/hash_graph.hpp" namespace vg { @@ -766,7 +765,11 @@ TEST_CASE("XdropAligner doesn't crash on a case where it is hard to find a seed" string graph_json = R"({"edge": [{"from": "92345167", "to": "92345168"}, {"from": "92345182", "to": "92345183"}, {"from": "92345165", "to": "92345166"}, {"from": "92345177", "to": "92345178"}, {"from": "92345171", "to": "92345172"}, {"from": "92345161", "to": "92345162"}, {"from": "92345183", "to": "92345184"}, {"from": "92345181", "to": "92345182"}, {"from": "92345178", "to": "92345179"}, {"from": "92345166", "to": "92345167"}, {"from": "92345179", "to": "92345180"}, {"from": "92345173", "to": "92345174"}, {"from": "92345184", "to": "92345185"}, {"from": "92345169", "to": "92345170"}, {"from": "92345185", "to": "92345186"}, {"from": "92345160", "to": "92345161"}, {"from": "92345174", "to": "92345175"}, {"from": "92345162", "to": "92345163"}, {"from": "92345175", "to": "92345176"}, {"from": "92345168", "to": "92345169"}, {"from": "92345163", "to": "92345164"}, {"from": "92345172", "to": "92345173"}, {"from": "92345180", "to": "92345181"}, {"from": "92345176", "to": "92345177"}, {"from": "92345170", "to": "92345171"}, {"from": "92345164", "to": "92345165"}], "node": [{"id": "92345167", "sequence": "TTTATATATATATATTTATATATATATATTTA"}, {"id": "92345182", "sequence": "TATATATATTTATATATATATTTATATATATA"}, {"id": "92345165", "sequence": "ATATATATATATTTATATATATTTATATATTA"}, {"id": "92345177", "sequence": "TTTATATATATATTTATATATATATATTATAT"}, {"id": "92345171", "sequence": "TTATATATATATTTATATATATATTTATATAT"}, {"id": "92345161", "sequence": "ATATATTTATATATTTTTATATATTATATATT"}, {"id": "92345183", "sequence": "TTTATATATATTTATATATATATTTATATATA"}, {"id": "92345181", "sequence": "ATATATTATATATATATTTATATATATATTTA"}, {"id": "92345178", "sequence": "ATATATTTATATATATATTTATATATATATTT"}, {"id": "92345166", "sequence": "TTTATATATATTTATATATATATTTATATATA"}, {"id": "92345179", "sequence": "ATATATATATTTATATATATATTTATATATAT"}, {"id": "92345173", "sequence": "ATATTTATATATATATATTTATATATATATTT"}, {"id": "92345184", "sequence": "TATTTATATATATATTTATATATATTTATATA"}, {"id": "92345169", "sequence": "TTTATATATATATTTATATATATATTTATATA"}, {"id": "92345185", "sequence": "TATATTTATATATATATATATATATTTATATA"}, {"id": "92345160", "sequence": "ATTTATATATATATTTATATATATATTTATAT"}, {"id": "92345174", "sequence": "ATATATATATTTATATATATATTATTTATATA"}, {"id": "92345162", "sequence": "TATATATATATTTATATATTATATATATATTT"}, {"id": "92345175", "sequence": "TATATTTATATATATATTATATATATATTTAT"}, {"id": "92345168", "sequence": "TATATATATTTATATATATATTTATATATATA"}, {"id": "92345163", "sequence": "ATATATTTATATATATATTTATATATATTTAT"}, {"id": "92345172", "sequence": "ATATATATATATTTATATATATATTTATATAT"}, {"id": "92345180", "sequence": "ATTTATATATATATTTATATATATATTTATAT"}, {"id": "92345176", "sequence": "ATATATATATTATATATATATTTATATATATA"}, {"id": "92345170", "sequence": "TATATTTATATATATATATTATATATATATAT"}, {"id": "92345164", "sequence": "ATATATATTTATATATATTTATATATATATTT"}, {"id": "92345186", "sequence": "TATATTTATATATATTTATATATATATTTATA"}]})"; - auto graph = json_to_graph(graph_json); + Graph source; + json2pb(source, graph_json.c_str(), graph_json.size()); + + VG graph; + graph.extend(source); Alignment aln; aln.set_sequence("CAGCACTTTGGGAGGCCAAGGTGGGTGGATCATCTGAGGTCAGGAGTTTGAGACCAGCCTGACCAACATGGTGAAATCCTGTCTCTACTGAAAATACTAAAATTAGCCAGGCGTGGCGGCCAGTGCCTGTAATCCCGGCTACTGGGGAGG"); diff --git a/src/unittest/xg.cpp b/src/unittest/xg.cpp index fbe0e8ff1e..d74db5d0b0 100644 --- a/src/unittest/xg.cpp +++ b/src/unittest/xg.cpp @@ -9,7 +9,6 @@ #include "xg.hpp" #include "graph.hpp" #include "algorithms/subgraph.hpp" -#include "support/json.hpp" #include namespace vg { @@ -24,9 +23,13 @@ TEST_CASE("We can build an xg index on a nice graph", "[xg]") { "edge":[{"to":2,"from":1}]} )"; - // Load the JSON and build the xg index + // Load the JSON + Graph proto_graph; + json2pb(proto_graph, graph_json.c_str(), graph_json.size()); + + // Build the xg index xg::XG xg_index; - xg_index.from_path_handle_graph(*json_to_graph(graph_json)); + xg_index.from_path_handle_graph(VG(proto_graph)); VG vg_graph; algorithms::extract_context(xg_index, vg_graph, xg_index.get_handle(1), 0, 100); @@ -47,9 +50,13 @@ TEST_CASE("We can build an xg index on a nasty graph", "[xg]") { "edge":[{"to":2,"from":1}]} )"; - // Load the JSON and build the xg index + // Load the JSON + Graph proto_graph; + json2pb(proto_graph, graph_json.c_str(), graph_json.size()); + + // Build the xg index xg::XG xg_index; - xg_index.from_path_handle_graph(*json_to_graph(graph_json)); + xg_index.from_path_handle_graph(VG(proto_graph)); VG vg_graph; algorithms::extract_context(xg_index, vg_graph, xg_index.get_handle(1), 0, 100); @@ -162,7 +169,7 @@ TEST_CASE("We can build an xg index on a very nasty graph", "[xg]") { sort_by_id_dedup_and_clean(proto_graph); // Build the xg index xg::XG xg_index; - xg_index.from_path_handle_graph(VG(proto_graph)); // TODO: replace with json_to_graph() once proto_graph usage is refactored + xg_index.from_path_handle_graph(VG(proto_graph)); SECTION("Context extraction gets something") { VG graph; @@ -299,7 +306,7 @@ TEST_CASE("We can build the xg index on a small graph with discontinuous node id sort_by_id_dedup_and_clean(proto_graph); // Build the xg index xg::XG xg_index; - xg_index.from_path_handle_graph(VG(proto_graph)); // TODO: replace with json_to_graph() once proto_graph usage is refactored + xg_index.from_path_handle_graph(VG(proto_graph)); VG vg_graph; algorithms::extract_context(xg_index, vg_graph, xg_index.get_handle(10), 0, 100); @@ -320,9 +327,13 @@ TEST_CASE("Looping over XG handles in parallel works", "[xg]") { "edge":[{"to":2,"from":1}]} )"; - // Load the JSON and build the xg index + // Load the JSON + Graph proto_graph; + json2pb(proto_graph, graph_json.c_str(), graph_json.size()); + + // Build the xg index xg::XG xg_index; - xg_index.from_path_handle_graph(*json_to_graph(graph_json)); + xg_index.from_path_handle_graph(VG(proto_graph)); size_t count = 0; From 695cff5fc1c41a6e246c7bf26c964451be21540d Mon Sep 17 00:00:00 2001 From: Adam Novak Date: Tue, 10 Feb 2026 15:20:42 -0500 Subject: [PATCH 20/77] Replace string_to_graph with json2graph --- src/unittest/vg.cpp | 47 ++++++++++++++++++--------------------------- 1 file changed, 19 insertions(+), 28 deletions(-) diff --git a/src/unittest/vg.cpp b/src/unittest/vg.cpp index 9beb3e1ca7..b2795b57cc 100644 --- a/src/unittest/vg.cpp +++ b/src/unittest/vg.cpp @@ -8,6 +8,7 @@ #include "../utility.hpp" #include "../algorithms/normalize.hpp" #include "../algorithms/disjoint_components.hpp" +#include "../io/json2graph.hpp" #include "handle.hpp" namespace vg { @@ -15,16 +16,6 @@ namespace unittest { using namespace std; -// Turn a JSON string into a VG graph -VG string_to_graph(const string& json) { - VG graph; - Graph chunk; - json2pb(chunk, json.c_str(), json.size()); - graph.merge(chunk); - - return graph; -} - TEST_CASE("dagify() should render the graph acyclic", "[vg][cycles][dagify]") { unordered_map > node_translation; @@ -44,7 +35,7 @@ TEST_CASE("dagify() should render the graph acyclic", "[vg][cycles][dagify]") { )"; - VG graph = string_to_graph(graph_json); + VG graph; vg::io::json2graph(graph_json, &graph); VG dag = graph.dagify(5, node_translation, 5, 0); @@ -69,7 +60,7 @@ TEST_CASE("dagify() should render the graph acyclic", "[vg][cycles][dagify]") { )"; - VG graph = string_to_graph(graph_json); + VG graph; vg::io::json2graph(graph_json, &graph); VG dag = graph.dagify(5, node_translation, 5, 0); @@ -93,7 +84,7 @@ TEST_CASE("dagify() should render the graph acyclic", "[vg][cycles][dagify]") { )"; - VG graph = string_to_graph(graph_json); + VG graph; vg::io::json2graph(graph_json, &graph); VG dag = graph.dagify(5, node_translation, 5, 0); @@ -123,7 +114,7 @@ TEST_CASE("unfold() should properly unfold a graph out to the requested length", } )"; - VG graph = string_to_graph(graph_json); + VG graph; vg::io::json2graph(graph_json, &graph); unordered_map > node_translation; VG unfolded = graph.unfold(10000, node_translation); @@ -252,7 +243,7 @@ TEST_CASE("unfold() should properly unfold a graph out to the requested length", } )"; - VG graph = string_to_graph(graph_json); + VG graph; vg::io::json2graph(graph_json, &graph); unordered_map > node_translation; VG unfolded = graph.unfold(10000, node_translation); @@ -327,7 +318,7 @@ TEST_CASE("unfold() should properly unfold a graph out to the requested length", } )"; - VG graph = string_to_graph(graph_json); + VG graph; vg::io::json2graph(graph_json, &graph); unordered_map > node_translation; VG unfolded = graph.unfold(10000, node_translation); @@ -417,7 +408,7 @@ TEST_CASE("unfold() should properly unfold a graph out to the requested length", } )"; - VG graph = string_to_graph(graph_json); + VG graph; vg::io::json2graph(graph_json, &graph); unordered_map > node_translation; VG unfolded = graph.unfold(10000, node_translation); @@ -574,7 +565,7 @@ TEST_CASE("unfold() should properly unfold a graph out to the requested length", } )"; - VG graph = string_to_graph(graph_json); + VG graph; vg::io::json2graph(graph_json, &graph); unordered_map > node_translation; VG unfolded = graph.unfold(10000, node_translation); @@ -742,7 +733,7 @@ TEST_CASE("unfold() should properly unfold a graph out to the requested length", } )"; - VG graph = string_to_graph(graph_json); + VG graph; vg::io::json2graph(graph_json, &graph); unordered_map > node_translation; VG unfolded = graph.unfold(2, node_translation); @@ -904,7 +895,7 @@ TEST_CASE("expand_context_by_length() should respect barriers", "[vg][context]") } )"; - VG graph = string_to_graph(graph_json); + VG graph; vg::io::json2graph(graph_json, &graph); SECTION("barriers on either end of the seed node should stop anything being extracted") { @@ -962,7 +953,7 @@ TEST_CASE("add_nodes_and_edges() should connect all nodes", "[vg][edit]") { )"; // Define a graph - VG graph = string_to_graph(graph_json); + VG graph; vg::io::json2graph(graph_json, &graph); const string path_json = R"( { @@ -1051,7 +1042,7 @@ TEST_CASE("edit() should not get confused even under very confusing circumstance )"; // Define a graph - VG graph = string_to_graph(graph_json); + VG graph; vg::io::json2graph(graph_json, &graph); // And a path that doubles back on itself through an edge that isn't in the graph yet const string path_json = R"( @@ -1310,7 +1301,7 @@ TEST_CASE("normalize() can join nodes and merge siblings", "[vg][normalize]") { )"; - VG graph = string_to_graph(graph_json); + VG graph; vg::io::json2graph(graph_json, &graph); algorithms::normalize(&graph); // One of the two alternative Ts should have been eliminated @@ -1341,7 +1332,7 @@ TEST_CASE("normalize() can join nodes and merge siblings", "[vg][normalize]") { )"; - VG graph = string_to_graph(graph_json); + VG graph; vg::io::json2graph(graph_json, &graph); algorithms::normalize(&graph); // Those duplicate Ts should be eliminated @@ -1375,7 +1366,7 @@ TEST_CASE("normalize() can join nodes and merge siblings", "[vg][normalize]") { )"; - VG graph = string_to_graph(graph_json); + VG graph; vg::io::json2graph(graph_json, &graph); algorithms::normalize(&graph); // Those duplicate Ts and Gs should be eliminated @@ -1409,7 +1400,7 @@ TEST_CASE("normalize() can join nodes and merge siblings", "[vg][normalize]") { )"; - VG graph = string_to_graph(graph_json); + VG graph; vg::io::json2graph(graph_json, &graph); algorithms::normalize(&graph); // Those duplicate Ts and Gs should be eliminated @@ -1447,7 +1438,7 @@ TEST_CASE("normalize() can join nodes and merge siblings when nodes are backward )"; - VG graph = string_to_graph(graph_json); + VG graph; vg::io::json2graph(graph_json, &graph); algorithms::normalize(&graph); // Those duplicate Ts (actually As) should be eliminated @@ -1486,7 +1477,7 @@ TEST_CASE("normalize() can join nodes and merge siblings when nodes are backward )"; - VG graph = string_to_graph(graph_json); + VG graph; vg::io::json2graph(graph_json, &graph); algorithms::normalize(&graph); // Those duplicate Ts (actually As) and Gs (actually Cs) should be eliminated From 8ede8df834815db758be32396c2c8de9ec3068d1 Mon Sep 17 00:00:00 2001 From: Adam Novak Date: Tue, 10 Feb 2026 17:19:24 -0500 Subject: [PATCH 21/77] Remove a bunch of mostly unused functions for working with Protobuf Graph objects --- src/graph.cpp | 87 --------------------------------------------------- src/graph.hpp | 33 ------------------- 2 files changed, 120 deletions(-) diff --git a/src/graph.cpp b/src/graph.cpp index beca52b5e1..3f23ffef18 100644 --- a/src/graph.cpp +++ b/src/graph.cpp @@ -2,93 +2,6 @@ namespace vg { -void sort_by_id_dedup_and_clean(Graph& graph) { - remove_duplicates(graph); // graph is sorted here - remove_orphan_edges(graph); -} - -void remove_duplicates(Graph& graph) { - remove_duplicate_nodes(graph); - remove_duplicate_edges(graph); -} - -void remove_duplicate_edges(Graph& graph) { - sort_edges_by_id(graph); - graph.mutable_edge()->erase(std::unique(graph.mutable_edge()->begin(), - graph.mutable_edge()->end(), - [](const Edge& a, const Edge& b) { - return make_tuple(a.from(), a.to(), a.from_start(), a.to_end()) - == make_tuple(b.from(), b.to(), b.from_start(), b.to_end()); - }), graph.mutable_edge()->end()); - -} - -void remove_duplicate_nodes(Graph& graph) { - sort_nodes_by_id(graph); - graph.mutable_node()->erase(std::unique(graph.mutable_node()->begin(), - graph.mutable_node()->end(), - [](const Node& a, const Node& b) { - return a.id() == b.id(); - }), graph.mutable_node()->end()); -} - -void remove_orphan_edges(Graph& graph) { - set ids; - for (auto& node : graph.node()) { - ids.insert(node.id()); - } - graph.mutable_edge()->erase(std::remove_if(graph.mutable_edge()->begin(), - graph.mutable_edge()->end(), - [&ids](const Edge& e) { - return !ids.count(e.from()) || !ids.count(e.to()); - }), graph.mutable_edge()->end()); -} - -void sort_by_id(Graph& graph) { - sort_nodes_by_id(graph); - sort_edges_by_id(graph); -} - -void sort_nodes_by_id(Graph& graph) { - std::sort(graph.mutable_node()->begin(), - graph.mutable_node()->end(), - [](const Node& a, const Node& b) { - return a.id() < b.id(); - }); -} - -void sort_edges_by_id(Graph& graph) { - std::sort(graph.mutable_edge()->begin(), - graph.mutable_edge()->end(), - [](const Edge& a, const Edge& b) { - return make_tuple(a.from(), a.to(), a.from_start(), a.to_end()) - < make_tuple(b.from(), b.to(), b.from_start(), b.to_end()); - }); -} - -bool is_id_sortable(const Graph& graph) { - for (auto& edge : graph.edge()) { - if (edge.from() >= edge.to()) return false; - } - return true; -} - -bool has_inversion(const Graph& graph) { - for (auto& edge : graph.edge()) { - if (edge.from_start() || edge.to_end()) return true; - } - return false; -} - -void flip_doubly_reversed_edges(Graph& graph) { - for (auto& edge : *graph.mutable_edge()) { - if (edge.from_start() && edge.to_end()) { - edge.set_from_start(false); - edge.set_to_end(false); - } - } -} - void from_handle_graph(const HandleGraph& from, Graph& to) { from.for_each_handle([&](const handle_t& h) { Node* node = to.add_node(); diff --git a/src/graph.hpp b/src/graph.hpp index 964e46cceb..c85afe88ab 100644 --- a/src/graph.hpp +++ b/src/graph.hpp @@ -11,39 +11,6 @@ namespace vg { using namespace std; -/// remove duplicates and sort by id -void sort_by_id_dedup_and_clean(Graph& graph); - -/// remove duplicate nodes and edges -void remove_duplicates(Graph& graph); - -/// remove duplicate edges -void remove_duplicate_edges(Graph& graph); - -/// remove duplicate nodes -void remove_duplicate_nodes(Graph& graph); - -/// remove edges that link to a node that is not in the graph -void remove_orphan_edges(Graph& graph); - -/// order the nodes and edges in the graph by id -void sort_by_id(Graph& graph); - -/// order the nodes in the graph by id -void sort_nodes_by_id(Graph& graph); - -/// order the edges in the graph by id pairs -void sort_edges_by_id(Graph& graph); - -/// returns true if the graph is id-sortable (no reverse links) -bool is_id_sortable(const Graph& graph); - -/// returns true if we find an edge that may specify an inversion -bool has_inversion(const Graph& graph); - -/// clean up doubly-reversed edges -void flip_doubly_reversed_edges(Graph& graph); - // transfer data from a HandleGraph into an empty Graph void from_handle_graph(const HandleGraph& from, Graph& to); From e47279966f8cc242a42145e6c1d6aeffe490f598 Mon Sep 17 00:00:00 2001 From: Adam Novak Date: Tue, 10 Feb 2026 17:20:50 -0500 Subject: [PATCH 22/77] Mostly-automatically convert tests to use vg::io::json2graph --- src/unittest/banded_global_aligner.cpp | 9 +- src/unittest/cactus.cpp | 28 +- src/unittest/chunker.cpp | 12 +- src/unittest/copy_graph.cpp | 179 ++++++------ src/unittest/dijkstra.cpp | 14 +- src/unittest/gbwt_extender.cpp | 9 +- src/unittest/genotypekit.cpp | 318 +++++++++------------ src/unittest/genotyper.cpp | 14 +- src/unittest/haplotypes.cpp | 50 ++-- src/unittest/indexed_vg.cpp | 2 +- src/unittest/mapper.cpp | 33 +-- src/unittest/minimizer_mapper.cpp | 67 ++--- src/unittest/multipath_alignment_graph.cpp | 13 +- src/unittest/multipath_mapper.cpp | 64 ++--- src/unittest/path_component_index.cpp | 13 +- src/unittest/path_index.cpp | 74 ++--- src/unittest/phase_unfolder.cpp | 63 ++-- src/unittest/readfilter.cpp | 13 +- src/unittest/sampler.cpp | 45 +-- src/unittest/snarl_distance_index.cpp | 54 +--- src/unittest/snarls.cpp | 120 ++++---- src/unittest/source_sink_overlay.cpp | 11 +- src/unittest/variant_adder.cpp | 90 ++---- src/unittest/vg_algorithms.cpp | 35 +-- src/unittest/vpkg.cpp | 54 ++-- src/unittest/xdrop_aligner.cpp | 11 +- src/unittest/xg.cpp | 71 +++-- 27 files changed, 608 insertions(+), 858 deletions(-) diff --git a/src/unittest/banded_global_aligner.cpp b/src/unittest/banded_global_aligner.cpp index 045e9bfa97..6b5fb4b3c8 100644 --- a/src/unittest/banded_global_aligner.cpp +++ b/src/unittest/banded_global_aligner.cpp @@ -10,7 +10,7 @@ #include "vg.hpp" #include "path.hpp" #include "banded_global_aligner.hpp" -#include "vg/io/json2pb.h" +#include "../io/json2graph.hpp" #include "bdsg/hash_graph.hpp" #include "../algorithms/pad_band.hpp" @@ -3515,10 +3515,9 @@ namespace vg { SECTION( "Banded global aligner does not produce empty edits when there is an insertion an empty node") { string graph_json = R"({"edge": [{"to_end": true, "from_start": true, "to": 22, "from": 20}, {"to": 26, "from": 20}, {"to": 24, "from": 20}, {"to_end": true, "from_start": true, "to": 26, "from": 4}, {"to_end": true, "from_start": true, "to": 24, "from": 4}], "node": [{"sequence": "C", "id": 24}, {"sequence": "GAGA", "id": 20}, {"sequence": "T", "id": 26}, {"sequence": "GGAGTCT", "id": 4}, {"id": 22}]})"; - - Graph graph; - json2pb(graph, graph_json.c_str(), graph_json.size()); - VG vg_graph(graph); + + bdsg::HashGraph vg_graph; + vg::io::json2graph(graph_json, &vg_graph); TestAligner aligner_source; const Aligner& aligner = *aligner_source.get_regular_aligner(); diff --git a/src/unittest/cactus.cpp b/src/unittest/cactus.cpp index 7447ee247d..5e518db4ef 100644 --- a/src/unittest/cactus.cpp +++ b/src/unittest/cactus.cpp @@ -5,8 +5,9 @@ #include #include -#include "vg/io/json2pb.h" +#include "../io/json2graph.hpp" #include "../cactus.hpp" +#include #include "catch.hpp" namespace vg { @@ -14,9 +15,7 @@ namespace unittest { using namespace std; TEST_CASE("We can convert a two-tailed graph to Cactus", "[cactus]") { - - VG graph; - + string graph_json = R"( {"node":[{"sequence":"GT","id":7575}, {"sequence":"TGTTAACAGCACAACATTTA","id":7580}, @@ -25,20 +24,18 @@ TEST_CASE("We can convert a two-tailed graph to Cactus", "[cactus]") { "edge":[{"from":7575,"to":7580,"from_start":true}, {"from":7575,"to":7576}]} )"; - - Graph g; - json2pb(g, graph_json.c_str(), graph_json.size()); - graph.extend(g); - // Make sure we can make a Cactus graph and get something out. + bdsg::HashGraph graph; + vg::io::json2graph(graph_json, &graph); + + // Make sure we can make a Cactus graph and get something out. auto cactusified = cactusify(graph); REQUIRE(cactusified.is_valid()); } TEST_CASE("We can convert a hairpin graph to Cactus", "[cactus]") { - VG graph; - + // Here's a graph where only the left side of node 2 is dangling, and the right side of node 1 has a self loop. string graph_json = R"( {"node": [{"sequence": "A", "id": 1}, @@ -46,12 +43,11 @@ TEST_CASE("We can convert a hairpin graph to Cactus", "[cactus]") { "edge": [{"from": 2, "to": 1}, {"from": 1, "to": 1, "to_end": true}]} )"; - - Graph g; - json2pb(g, graph_json.c_str(), graph_json.size()); - graph.extend(g); - // Make sure we can make a Cactus graph and get something out. + bdsg::HashGraph graph; + vg::io::json2graph(graph_json, &graph); + + // Make sure we can make a Cactus graph and get something out. auto cactusified = cactusify(graph); REQUIRE(cactusified.is_valid()); } diff --git a/src/unittest/chunker.cpp b/src/unittest/chunker.cpp index 24f7d3b645..3be2298c15 100644 --- a/src/unittest/chunker.cpp +++ b/src/unittest/chunker.cpp @@ -7,6 +7,8 @@ #include "vg.hpp" #include "xg.hpp" #include "path.hpp" +#include "../io/json2graph.hpp" +#include namespace vg { namespace unittest { @@ -83,13 +85,13 @@ TEST_CASE("basic graph chunking", "[chunk]") { )"; - // Load it into Protobuf - Graph chunk; - json2pb(chunk, graph_json.c_str(), graph_json.size()); - + // Load the graph + bdsg::HashGraph graph; + vg::io::json2graph(graph_json, &graph); + // Pass it over to XG xg::XG index; - index.from_path_handle_graph(VG(chunk)); + index.from_path_handle_graph(graph); PathChunker chunker(&index); diff --git a/src/unittest/copy_graph.cpp b/src/unittest/copy_graph.cpp index 581b683130..4e7e878075 100644 --- a/src/unittest/copy_graph.cpp +++ b/src/unittest/copy_graph.cpp @@ -1,6 +1,7 @@ #include "catch.hpp" #include "../handle.hpp" #include "../vg.hpp" +#include "../io/json2graph.hpp" #include "xg.hpp" #include "bdsg/packed_graph.hpp" @@ -53,14 +54,15 @@ namespace vg { ] } )"; - Graph proto_graph; - json2pb(proto_graph, graph_json.c_str(), graph_json.size()); - + + bdsg::HashGraph source; + vg::io::json2graph(graph_json, &source); + xg::XG xg; - xg.from_path_handle_graph(VG(proto_graph)); + xg.from_path_handle_graph(source); VG vg; handlealgs::copy_handle_graph(&xg, &vg); - + REQUIRE(xg.get_node_count() == 1); REQUIRE(vg.get_node_count() == 1); } @@ -72,14 +74,15 @@ namespace vg { ] } )"; - Graph proto_graph; - json2pb(proto_graph, graph_json.c_str(), graph_json.size()); - + + bdsg::HashGraph source; + vg::io::json2graph(graph_json, &source); + xg::XG xg; - xg.from_path_handle_graph(VG(proto_graph)); + xg.from_path_handle_graph(source); bdsg::PackedGraph pg; handlealgs::copy_handle_graph(&xg, &pg); - + REQUIRE(xg.get_node_count() == 1); REQUIRE(pg.get_node_count() == 1); } @@ -91,14 +94,15 @@ namespace vg { ] } )"; - Graph proto_graph; - json2pb(proto_graph, graph_json.c_str(), graph_json.size()); - + + bdsg::HashGraph source; + vg::io::json2graph(graph_json, &source); + xg::XG xg; - xg.from_path_handle_graph(VG(proto_graph)); + xg.from_path_handle_graph(source); bdsg::HashGraph hg; handlealgs::copy_handle_graph(&xg, &hg); - + REQUIRE(xg.get_node_count() == 1); REQUIRE(hg.get_node_count() == 1); } @@ -120,19 +124,20 @@ namespace vg { ] } )"; - Graph proto_graph; - json2pb(proto_graph, graph_json.c_str(), graph_json.size()); - + + bdsg::HashGraph source; + vg::io::json2graph(graph_json, &source); + xg::XG xg; - xg.from_path_handle_graph(VG(proto_graph)); + xg.from_path_handle_graph(source); VG vg; handlealgs::copy_handle_graph(&xg, &vg); - + REQUIRE(xg.get_node_count() == 4); REQUIRE(vg.get_node_count() == 4); REQUIRE(vg.edge_count() == 4); REQUIRE(vg.length() == 16); - + } TEST_CASE( "copy_handle_graph converter works on graphs with one reversing edge, xg to pg", "[handle][pg][xg]") { string graph_json = R"( @@ -151,14 +156,15 @@ namespace vg { ] } )"; - Graph proto_graph; - json2pb(proto_graph, graph_json.c_str(), graph_json.size()); - + + bdsg::HashGraph source; + vg::io::json2graph(graph_json, &source); + xg::XG xg; - xg.from_path_handle_graph(VG(proto_graph)); + xg.from_path_handle_graph(source); bdsg::PackedGraph pg; handlealgs::copy_handle_graph(&xg, &pg); - + REQUIRE(xg.get_node_count() == 4); REQUIRE(pg.get_node_count() == 4); @@ -168,14 +174,14 @@ namespace vg { return true; }); REQUIRE(length == 16); - + int edge_count = 0; pg.for_each_edge([&](const edge_t& edge) { edge_count += 1; return true; }); REQUIRE(edge_count == 4); - + } TEST_CASE( "copy_handle_graph converter works on graphs with one reversing edge, xg to hg", "[handle][hg][xg]") { string graph_json = R"( @@ -194,14 +200,15 @@ namespace vg { ] } )"; - Graph proto_graph; - json2pb(proto_graph, graph_json.c_str(), graph_json.size()); - + + bdsg::HashGraph source; + vg::io::json2graph(graph_json, &source); + xg::XG xg; - xg.from_path_handle_graph(VG(proto_graph)); + xg.from_path_handle_graph(source); bdsg::HashGraph hg; handlealgs::copy_handle_graph(&xg, &hg); - + REQUIRE(xg.get_node_count() == 4); REQUIRE(hg.get_node_count() == 4); int length = 0; @@ -210,14 +217,14 @@ namespace vg { return true; }); REQUIRE(length == 16); - + int edge_count = 0; hg.for_each_edge([&](const edge_t& edge) { edge_count += 1; return true; }); REQUIRE(edge_count == 4); - + } TEST_CASE( "copy_handle_graph converter works on graphs with reversing edges and loops", "[handle][vg][xg]") { string graph_json = R"( @@ -239,14 +246,15 @@ namespace vg { ] } )"; - Graph proto_graph; - json2pb(proto_graph, graph_json.c_str(), graph_json.size()); - + + bdsg::HashGraph source; + vg::io::json2graph(graph_json, &source); + xg::XG xg; - xg.from_path_handle_graph(VG(proto_graph)); + xg.from_path_handle_graph(source); VG vg; handlealgs::copy_handle_graph(&xg, &vg); - + REQUIRE(xg.get_sequence(xg.get_handle(1)) == "GATT"); REQUIRE(xg.get_sequence(xg.get_handle(3)) == "CGAT"); REQUIRE(xg.get_node_count() == 4); @@ -274,26 +282,27 @@ namespace vg { ] } )"; - Graph proto_graph; - json2pb(proto_graph, graph_json.c_str(), graph_json.size()); - + + bdsg::HashGraph source; + vg::io::json2graph(graph_json, &source); + xg::XG xg; - xg.from_path_handle_graph(VG(proto_graph)); + xg.from_path_handle_graph(source); bdsg::PackedGraph pg; handlealgs::copy_handle_graph(&xg, &pg); - + REQUIRE(xg.get_sequence(xg.get_handle(1)) == "GATT"); REQUIRE(xg.get_sequence(xg.get_handle(3)) == "CGAT"); REQUIRE(xg.get_node_count() == 4); REQUIRE(pg.get_node_count() == 4); - + int length = 0; pg.for_each_handle([&](const handle_t& here) { length += pg.get_length(here); return true; }); REQUIRE(length == 16); - + int edge_count = 0; pg.for_each_edge([&](const edge_t& edge) { edge_count += 1; @@ -321,26 +330,27 @@ namespace vg { ] } )"; - Graph proto_graph; - json2pb(proto_graph, graph_json.c_str(), graph_json.size()); - + + bdsg::HashGraph source; + vg::io::json2graph(graph_json, &source); + xg::XG xg; - xg.from_path_handle_graph(VG(proto_graph)); + xg.from_path_handle_graph(source); bdsg::HashGraph hg; handlealgs::copy_handle_graph(&xg, &hg); - + REQUIRE(xg.get_sequence(xg.get_handle(1)) == "GATT"); REQUIRE(xg.get_sequence(xg.get_handle(3)) == "CGAT"); REQUIRE(xg.get_node_count() == 4); REQUIRE(hg.get_node_count() == 4); - + int length = 0; hg.for_each_handle([&](const handle_t& here) { length += hg.get_length(here); return true; }); REQUIRE(length == 16); - + int edge_count = 0; hg.for_each_edge([&](const edge_t& edge) { edge_count += 1; @@ -382,16 +392,17 @@ namespace vg { ] } )"; - Graph proto_graph; - json2pb(proto_graph, graph_json.c_str(), graph_json.size()); - + + bdsg::HashGraph source; + vg::io::json2graph(graph_json, &source); + xg::XG xg; - xg.from_path_handle_graph(VG(proto_graph)); + xg.from_path_handle_graph(source); VG vg; handlealgs::copy_path_handle_graph(&xg, &vg); - - + + REQUIRE(xg.get_sequence(xg.get_handle(1)) == "GATT"); REQUIRE(xg.get_sequence(xg.get_handle(3)) == "CGAT"); REQUIRE(xg.get_node_count() == 4); @@ -444,37 +455,38 @@ namespace vg { ] } )"; - Graph proto_graph; - json2pb(proto_graph, graph_json.c_str(), graph_json.size()); - + + bdsg::HashGraph source; + vg::io::json2graph(graph_json, &source); + xg::XG xg; - xg.from_path_handle_graph(VG(proto_graph)); + xg.from_path_handle_graph(source); bdsg::PackedGraph pg; handlealgs::copy_path_handle_graph(&xg, &pg); - - - + + + REQUIRE(xg.get_sequence(xg.get_handle(1)) == "GATT"); REQUIRE(xg.get_sequence(xg.get_handle(3)) == "CGAT"); REQUIRE(xg.get_node_count() == 4); REQUIRE(pg.get_node_count() == 4); - + int length = 0; pg.for_each_handle([&](const handle_t& here) { length += pg.get_length(here); return true; }); REQUIRE(length == 16); - + int edge_count = 0; pg.for_each_edge([&](const edge_t& edge) { edge_count += 1; return true; }); REQUIRE(edge_count == 7); - - + + REQUIRE(pg.has_path("path1") == true); REQUIRE(pg.has_path("path2") == true); REQUIRE(pg.get_path_count() == 2); @@ -521,37 +533,38 @@ namespace vg { ] } )"; - Graph proto_graph; - json2pb(proto_graph, graph_json.c_str(), graph_json.size()); - + + bdsg::HashGraph source; + vg::io::json2graph(graph_json, &source); + xg::XG xg; - xg.from_path_handle_graph(VG(proto_graph)); + xg.from_path_handle_graph(source); bdsg::HashGraph hg; handlealgs::copy_path_handle_graph(&xg, &hg); - - - + + + REQUIRE(xg.get_sequence(xg.get_handle(1)) == "GATT"); REQUIRE(xg.get_sequence(xg.get_handle(3)) == "CGAT"); REQUIRE(xg.get_node_count() == 4); REQUIRE(hg.get_node_count() == 4); - - + + int length = 0; hg.for_each_handle([&](const handle_t& here) { length += hg.get_length(here); return true; }); REQUIRE(length == 16); - + int edge_count = 0; hg.for_each_edge([&](const edge_t& edge) { edge_count += 1; return true; }); REQUIRE(edge_count == 7); - - + + REQUIRE(hg.has_path("path1") == true); REQUIRE(hg.has_path("path2") == true); REQUIRE(hg.get_path_count() == 2); diff --git a/src/unittest/dijkstra.cpp b/src/unittest/dijkstra.cpp index 2608567153..4e94414040 100644 --- a/src/unittest/dijkstra.cpp +++ b/src/unittest/dijkstra.cpp @@ -6,7 +6,7 @@ #include #include #include "../handle.hpp" -#include "vg/io/json2pb.h" +#include "../io/json2graph.hpp" #include "../vg.hpp" #include "catch.hpp" @@ -125,14 +125,12 @@ TEST_CASE("Dijkstra search handles early stopping correctly", "[dijkstra][algori TEST_CASE("Dijkstra search works on a particular problem graph", "[dijkstra][algorithms]") { string graph_json = R"( -{"node":[{"sequence":"A","id":"2454530"},{"sequence":"AGTGCTGGAGAGGATGTGGAGAAATAGGAAC","id":"2454529"},{"sequence":"C","id":"2454532"},{"sequence":"TTTTACACTGTTGGTGGGACTGTAAA","id":"2454533"},{"sequence":"A","id":"2454527"},{"sequence":"C","id":"2454528"},{"sequence":"G","id":"2454531"},{"sequence":"C","id":"2454534"},{"sequence":"T","id":"2454535"},{"sequence":"GGGTAATAA","id":"2454526"},{"sequence":"TAGTTCAACCATTGTGGAAGACTGTGGCAATT","id":"2454536"}],"edge":[{"from":"2454530","to":"2454532"},{"from":"2454530","to":"2454533"},{"from":"2454529","to":"2454530"},{"from":"2454529","to":"2454531"},{"from":"2454532","to":"2454533"},{"from":"2454533","to":"2454534"},{"from":"2454533","to":"2454535"},{"from":"2454527","to":"2454529"},{"from":"2454528","to":"2454529"},{"from":"2454531","to":"2454532"},{"from":"2454531","to":"2454533"},{"from":"2454534","to":"2454536"},{"from":"2454535","to":"2454536"},{"from":"2454526","to":"2454527"},{"from":"2454526","to":"2454528"}],"path":[{"name":"21","mapping":[{"position":{"node_id":"2454526"},"edit":[{"from_length":9,"to_length":9}],"rank":"3049077"},{"position":{"node_id":"2454528"},"edit":[{"from_length":1,"to_length":1}],"rank":"3049078"},{"position":{"node_id":"2454529"},"edit":[{"from_length":31,"to_length":31}],"rank":"3049079"},{"position":{"node_id":"2454531"},"edit":[{"from_length":1,"to_length":1}],"rank":"3049080"},{"position":{"node_id":"2454532"},"edit":[{"from_length":1,"to_length":1}],"rank":"3049081"},{"position":{"node_id":"2454533"},"edit":[{"from_length":26,"to_length":26}],"rank":"3049082"},{"position":{"node_id":"2454535"},"edit":[{"from_length":1,"to_length":1}],"rank":"3049083"},{"position":{"node_id":"2454536"},"edit":[{"from_length":32,"to_length":32}],"rank":"3049084"}]}]} +{"node":[{"sequence":"A","id":"2454530"},{"sequence":"AGTGCTGGAGAGGATGTGGAGAAATAGGAAC","id":"2454529"},{"sequence":"C","id":"2454532"},{"sequence":"TTTTACACTGTTGGTGGGACTGTAAA","id":"2454533"},{"sequence":"A","id":"2454527"},{"sequence":"C","id":"2454528"},{"sequence":"G","id":"2454531"},{"sequence":"C","id":"2454534"},{"sequence":"T","id":"2454535"},{"sequence":"GGGTAATAA","id":"2454526"},{"sequence":"TAGTTCAACCATTGTGGAAGACTGTGGCAATT","id":"2454536"}],"edge":[{"from":"2454530","to":"2454532"},{"from":"2454530","to":"2454533"},{"from":"2454529","to":"2454530"},{"from":"2454529","to":"2454531"},{"from":"2454532","to":"2454533"},{"from":"2454533","to":"2454534"},{"from":"2454533","to":"2454535"},{"from":"2454527","to":"2454529"},{"from":"2454528","to":"2454529"},{"from":"2454531","to":"2454532"},{"from":"2454531","to":"2454533"},{"from":"2454534","to":"2454536"},{"from":"2454535","to":"2454536"},{"from":"2454526","to":"2454527"},{"from":"2454526","to":"2454528"}],"path":[{"name":"21","mapping":[{"position":{"node_id":"2454526"},"edit":[{"from_length":9,"to_length":9}],"rank":"3049077"},{"position":{"node_id":"2454528"},"edit":[{"from_length":1,"to_length":1}],"rank":"3049078"},{"position":{"node_id":"2454529"},"edit":[{"from_length":31,"to_length":31}],"rank":"3049079"},{"position":{"node_id":"2454531"},"edit":[{"from_length":1,"to_length":1}],"rank":"3049080"},{"position":{"node_id":"2454532"},"edit":[{"from_length":1,"to_length":1}],"rank":"3049081"},{"position":{"node_id":"2454533"},"edit":[{"from_length":26,"to_length":26}],"rank":"3049082"},{"position":{"node_id":"2454535"},"edit":[{"from_length":1,"to_length":1}],"rank":"3049083"},{"position":{"node_id":"2454536"},"edit":[{"from_length":32,"to_length":32}],"rank":"3049084"}]}]} )"; - - Graph g; - json2pb(g, graph_json); - - // Wrap the graph in a HandleGraph - VG graph(g); + + // Load the graph + HashGraph graph; + vg::io::json2graph(graph_json, &graph); // Decide where to start handle_t start = graph.get_handle(2454536, true); diff --git a/src/unittest/gbwt_extender.cpp b/src/unittest/gbwt_extender.cpp index d04a225fdb..4835fbd511 100644 --- a/src/unittest/gbwt_extender.cpp +++ b/src/unittest/gbwt_extender.cpp @@ -5,7 +5,7 @@ #include "../gbwt_extender.hpp" #include "../gbwt_helper.hpp" -#include "vg/io/json2pb.h" +#include "../io/json2graph.hpp" #include "../utility.hpp" #include "../vg.hpp" @@ -90,10 +90,9 @@ gbwt::GBWT build_gbwt_index() { // Build a GBWTGraph using the provided GBWT index. gbwtgraph::GBWTGraph build_gbwt_graph(const gbwt::GBWT& gbwt_index) { - Graph graph; - json2pb(graph, gapless_extender_graph.c_str(), gapless_extender_graph.size()); - VG vg_graph(graph); - return gbwtgraph::GBWTGraph(gbwt_index, vg_graph, nullptr); + bdsg::HashGraph graph; + vg::io::json2graph(gapless_extender_graph, &graph); + return gbwtgraph::GBWTGraph(gbwt_index, graph, nullptr); } void same_position(const Position& pos, const Position& correct) { diff --git a/src/unittest/genotypekit.cpp b/src/unittest/genotypekit.cpp index af9bc2a4d8..b5d460c59a 100644 --- a/src/unittest/genotypekit.cpp +++ b/src/unittest/genotypekit.cpp @@ -10,6 +10,8 @@ #include "../traversal_finder.hpp" #include "xg.hpp" #include "../haplotype_extracter.hpp" +#include "../io/json2graph.hpp" +#include namespace Catch { @@ -62,10 +64,10 @@ namespace vg { namespace unittest { TEST_CASE("sites can be found with Cactus", "[genotype]") { - + // Build a toy graph const string graph_json = R"( - + { "node": [ {"id": 1, "sequence": "G"}, @@ -90,7 +92,7 @@ TEST_CASE("sites can be found with Cactus", "[genotype]") { {"from": 6, "to": 8}, {"from": 7, "to": 9}, {"from": 8, "to": 9} - + ], "path": [ {"name": "hint", "mapping": [ @@ -101,14 +103,13 @@ TEST_CASE("sites can be found with Cactus", "[genotype]") { ]} ] } - + )"; - + // Make an actual graph + // Note: Using VG here because the test uses VG-specific methods like get_node() and get_edge() VG graph; - Graph chunk; - json2pb(chunk, graph_json.c_str(), graph_json.size()); - graph.merge(chunk); + vg::io::json2graph(graph_json, &graph); // Make a CactusSnarlFinder unique_ptr finder(new CactusSnarlFinder(graph)); @@ -196,10 +197,10 @@ TEST_CASE("sites can be found with Cactus", "[genotype]") { } TEST_CASE("sites can be found with the IntegratedSnarlFinder", "[genotype][integrated-snarl-finder]") { - + // Build a toy graph const string graph_json = R"( - + { "node": [ {"id": 1, "sequence": "G"}, @@ -224,7 +225,7 @@ TEST_CASE("sites can be found with the IntegratedSnarlFinder", "[genotype][integ {"from": 6, "to": 8}, {"from": 7, "to": 9}, {"from": 8, "to": 9} - + ], "path": [ {"name": "hint", "mapping": [ @@ -235,14 +236,13 @@ TEST_CASE("sites can be found with the IntegratedSnarlFinder", "[genotype][integ ]} ] } - + )"; - + // Make an actual graph + // Note: Using VG here because the test uses VG-specific methods like get_node() and get_edge() VG graph; - Graph chunk; - json2pb(chunk, graph_json.c_str(), graph_json.size()); - graph.merge(chunk); + vg::io::json2graph(graph_json, &graph); // Make an IntegratedSnarlFinder unique_ptr finder(new IntegratedSnarlFinder(graph)); @@ -329,7 +329,7 @@ TEST_CASE("sites can be found with the IntegratedSnarlFinder", "[genotype][integ } TEST_CASE("IntegratedSnarlFinder works when cactus graph contains back-to-back cycles along root path", "[genotype][integrated-snarl-finder]") { - + // Build a toy graph const string graph_json = R"( @@ -351,17 +351,15 @@ TEST_CASE("IntegratedSnarlFinder works when cactus graph contains back-to-back c {"from": 3, "to": 5}, {"from": 4, "to": 6}, {"from": 5, "to": 6} - + ] } )"; // Make an actual graph - VG graph; - Graph chunk; - json2pb(chunk, graph_json.c_str(), graph_json.size()); - graph.merge(chunk); + bdsg::HashGraph graph; + vg::io::json2graph(graph_json, &graph); // Make an IntegratedSnarlFinder unique_ptr finder(new IntegratedSnarlFinder(graph)); @@ -375,18 +373,16 @@ TEST_CASE("IntegratedSnarlFinder works when cactus graph contains back-to-back c } TEST_CASE("IntegratedSnarlFinder works on an all bridge edge Y graph with specific numbering", "[genotype][integrated-snarl-finder]") { - + // Build a toy graph const string graph_json = R"( {"node":[{"id":"2","sequence":"G"},{"id":"3","sequence":"G"},{"id":"4","sequence":"G"},{"id":"5","sequence":"G"},{"id":"6","sequence":"G"},{"id":"11","sequence":"G"}], - "edge":[{"from":"2","to":"3"},{"from":"3","to":"6"},{"from":"4","to":"5"},{"from":"5","to":"6"},{"from":"6","to":"11"}]} + "edge":[{"from":"2","to":"3"},{"from":"3","to":"6"},{"from":"4","to":"5"},{"from":"5","to":"6"},{"from":"6","to":"11"}]} )"; // Make an actual graph - VG graph; - Graph chunk; - json2pb(chunk, graph_json.c_str(), graph_json.size()); - graph.merge(chunk); + bdsg::HashGraph graph; + vg::io::json2graph(graph_json, &graph); // Make an IntegratedSnarlFinder unique_ptr finder(new IntegratedSnarlFinder(graph)); @@ -403,18 +399,16 @@ TEST_CASE("IntegratedSnarlFinder works on an all bridge edge Y graph with specif } TEST_CASE("IntegratedSnarlFinder roots correctly an all bridge edge Y graph with winning longest path", "[genotype][integrated-snarl-finder]") { - + // Build a toy graph const string graph_json = R"( {"node":[{"id":"2","sequence":"G"},{"id":"3","sequence":"G"},{"id":"4","sequence":"GG"},{"id":"5","sequence":"G"},{"id":"6","sequence":"G"},{"id":"11","sequence":"GG"}], - "edge":[{"from":"2","to":"3"},{"from":"3","to":"6"},{"from":"4","to":"5"},{"from":"5","to":"6"},{"from":"6","to":"11"}]} + "edge":[{"from":"2","to":"3"},{"from":"3","to":"6"},{"from":"4","to":"5"},{"from":"5","to":"6"},{"from":"6","to":"11"}]} )"; // Make an actual graph - VG graph; - Graph chunk; - json2pb(chunk, graph_json.c_str(), graph_json.size()); - graph.merge(chunk); + bdsg::HashGraph graph; + vg::io::json2graph(graph_json, &graph); // Make an IntegratedSnarlFinder unique_ptr finder(new IntegratedSnarlFinder(graph)); @@ -452,7 +446,7 @@ TEST_CASE("IntegratedSnarlFinder roots correctly an all bridge edge Y graph with } TEST_CASE("IntegratedSnarlFinder works when cactus graph contains longer back-to-back cycles along root path", "[genotype][integrated-snarl-finder]") { - + // Build a toy graph const string graph_json = R"( @@ -482,17 +476,15 @@ TEST_CASE("IntegratedSnarlFinder works when cactus graph contains longer back-to {"from": 32, "to": 5}, {"from": 4, "to": 6}, {"from": 5, "to": 6} - + ] } )"; // Make an actual graph - VG graph; - Graph chunk; - json2pb(chunk, graph_json.c_str(), graph_json.size()); - graph.merge(chunk); + bdsg::HashGraph graph; + vg::io::json2graph(graph_json, &graph); // Make an IntegratedSnarlFinder unique_ptr finder(new IntegratedSnarlFinder(graph)); @@ -506,50 +498,48 @@ TEST_CASE("IntegratedSnarlFinder works when cactus graph contains longer back-to } TEST_CASE("IntegratedSnarlFinder works on a complex bundle-y region with a nested snarl", "[genotype][integrated-snarl-finder]") { - + // Build a toy graph const string graph_json = R"( {"edge": [{"from": "129672", "to": "129673"}, - {"from": "129662", "to": "129663"}, - {"from": "129662", "to": "129664"}, - {"from": "129664", "to": "129665"}, - {"from": "129664", "to": "129666"}, - {"from": "129666", "to": "129668"}, - {"from": "129666", "to": "129669"}, - {"from": "129666", "to": "129667"}, - {"from": "129667", "to": "129668"}, - {"from": "129667", "to": "129669"}, - {"from": "129669", "to": "129670"}, - {"from": "129669", "to": "129673"}, - {"from": "129671", "to": "129672"}, - {"from": "129668", "to": "129670"}, - {"from": "129668", "to": "129673"}, - {"from": "129665", "to": "129668"}, - {"from": "129665", "to": "129669"}, - {"from": "129665", "to": "129667"}, - {"from": "129670", "to": "129671"}, - {"from": "129670", "to": "129672"}, - {"from": "129663", "to": "129665"}, - {"from": "129663", "to": "129666"}], - "node": [{"id": "129672", "sequence": "AT"}, - {"id": "129662", "sequence": "CAGGTCAAACTGTGAT"}, - {"id": "129664", "sequence": "T"}, - {"id": "129666", "sequence": "T"}, - {"id": "129667", "sequence": "G"}, - {"id": "129669", "sequence": "G"}, - {"id": "129671", "sequence": "T"}, - {"id": "129668", "sequence": "A"}, - {"id": "129665", "sequence": "A"}, - {"id": "129670", "sequence": "A"}, - {"id": "129673", "sequence": "ATATATATATACTTATTGTAAAAATCTTTAGA"}, + {"from": "129662", "to": "129663"}, + {"from": "129662", "to": "129664"}, + {"from": "129664", "to": "129665"}, + {"from": "129664", "to": "129666"}, + {"from": "129666", "to": "129668"}, + {"from": "129666", "to": "129669"}, + {"from": "129666", "to": "129667"}, + {"from": "129667", "to": "129668"}, + {"from": "129667", "to": "129669"}, + {"from": "129669", "to": "129670"}, + {"from": "129669", "to": "129673"}, + {"from": "129671", "to": "129672"}, + {"from": "129668", "to": "129670"}, + {"from": "129668", "to": "129673"}, + {"from": "129665", "to": "129668"}, + {"from": "129665", "to": "129669"}, + {"from": "129665", "to": "129667"}, + {"from": "129670", "to": "129671"}, + {"from": "129670", "to": "129672"}, + {"from": "129663", "to": "129665"}, + {"from": "129663", "to": "129666"}], + "node": [{"id": "129672", "sequence": "AT"}, + {"id": "129662", "sequence": "CAGGTCAAACTGTGAT"}, + {"id": "129664", "sequence": "T"}, + {"id": "129666", "sequence": "T"}, + {"id": "129667", "sequence": "G"}, + {"id": "129669", "sequence": "G"}, + {"id": "129671", "sequence": "T"}, + {"id": "129668", "sequence": "A"}, + {"id": "129665", "sequence": "A"}, + {"id": "129670", "sequence": "A"}, + {"id": "129673", "sequence": "ATATATATATACTTATTGTAAAAATCTTTAGA"}, {"id": "129663", "sequence": "G"}]} )"; // Make an actual graph - VG graph; - Graph chunk; - json2pb(chunk, graph_json.c_str(), graph_json.size()); - graph.merge(chunk); + bdsg::HashGraph graph; + vg::io::json2graph(graph_json, &graph); // Make an IntegratedSnarlFinder unique_ptr finder(new IntegratedSnarlFinder(graph)); @@ -579,23 +569,21 @@ TEST_CASE("IntegratedSnarlFinder works on a complex bundle-y region with a neste } TEST_CASE("CactusSnarlFinder safely handles a single node graph", "[genotype][cactus-snarl-finder]") { - + // Build a toy graph const string graph_json = R"( - + { "node": [ {"id": 1, "sequence": "GATTACA"} ] } - + )"; - + // Make an actual graph - VG graph; - Graph chunk; - json2pb(chunk, graph_json.c_str(), graph_json.size()); - graph.merge(chunk); + bdsg::HashGraph graph; + vg::io::json2graph(graph_json, &graph); // Make a CactusSnarlFinder unique_ptr finder(new CactusSnarlFinder(graph)); @@ -607,15 +595,13 @@ TEST_CASE("CactusSnarlFinder safely handles a single node graph", "[genotype][ca } TEST_CASE("IntegratedSnarlFinder safely handles a completely empty graph", "[genotype][integrated-snarl-finder]") { - + // Build a toy graph const string graph_json = "{}"; // Make an actual graph - VG graph; - Graph chunk; - json2pb(chunk, graph_json.c_str(), graph_json.size()); - graph.merge(chunk); + bdsg::HashGraph graph; + vg::io::json2graph(graph_json, &graph); // Make a IntegratedSnarlFinder unique_ptr finder(new IntegratedSnarlFinder(graph)); @@ -625,7 +611,7 @@ TEST_CASE("IntegratedSnarlFinder safely handles a completely empty graph", "[gen } TEST_CASE("IntegratedSnarlFinder safely handles a single node graph", "[genotype][integrated-snarl-finder]") { - + // Build a toy graph const string graph_json = R"( @@ -638,10 +624,8 @@ TEST_CASE("IntegratedSnarlFinder safely handles a single node graph", "[genotype )"; // Make an actual graph - VG graph; - Graph chunk; - json2pb(chunk, graph_json.c_str(), graph_json.size()); - graph.merge(chunk); + bdsg::HashGraph graph; + vg::io::json2graph(graph_json, &graph); // Make an IntegratedSnarlFinder unique_ptr finder(new IntegratedSnarlFinder(graph)); @@ -651,7 +635,7 @@ TEST_CASE("IntegratedSnarlFinder safely handles a single node graph", "[genotype } TEST_CASE("IntegratedSnarlFinder produces all the correct types of single-node chains", "[genotype][integrated-snarl-finder]") { - + // Build a toy graph const string graph_json = R"( @@ -673,10 +657,8 @@ TEST_CASE("IntegratedSnarlFinder produces all the correct types of single-node c )"; // Make an actual graph - VG graph; - Graph chunk; - json2pb(chunk, graph_json.c_str(), graph_json.size()); - graph.merge(chunk); + bdsg::HashGraph graph; + vg::io::json2graph(graph_json, &graph); // Make an IntegratedSnarlFinder IntegratedSnarlFinder finder(graph); @@ -736,7 +718,7 @@ TEST_CASE("IntegratedSnarlFinder produces all the correct types of single-node c } TEST_CASE("IntegratedSnarlFinder safely handles a path when forced to root at one end", "[genotype][integrated-snarl-finder]") { - + // Build a toy graph const string graph_json = R"( @@ -757,10 +739,8 @@ TEST_CASE("IntegratedSnarlFinder safely handles a path when forced to root at on )"; // Make an actual graph - VG graph; - Graph chunk; - json2pb(chunk, graph_json.c_str(), graph_json.size()); - graph.merge(chunk); + bdsg::HashGraph graph; + vg::io::json2graph(graph_json, &graph); // Make an IntegratedSnarlFinder unique_ptr finder(new IntegratedSnarlFinder(graph)); @@ -770,7 +750,7 @@ TEST_CASE("IntegratedSnarlFinder safely handles a path when forced to root at on } TEST_CASE("IntegratedSnarlFinder safely handles a single node connected component in a larger graph", "[genotype][integrated-snarl-finder]") { - + // Build a toy graph const string graph_json = R"( @@ -787,10 +767,8 @@ TEST_CASE("IntegratedSnarlFinder safely handles a single node connected componen )"; // Make an actual graph - VG graph; - Graph chunk; - json2pb(chunk, graph_json.c_str(), graph_json.size()); - graph.merge(chunk); + bdsg::HashGraph graph; + vg::io::json2graph(graph_json, &graph); // Make an IntegratedSnarlFinder unique_ptr finder(new IntegratedSnarlFinder(graph)); @@ -813,7 +791,7 @@ TEST_CASE("IntegratedSnarlFinder safely handles a single node connected componen } TEST_CASE("IntegratedSnarlFinder safely handles a single node cycle", "[genotype][integrated-snarl-finder]") { - + // Build a toy graph const string graph_json = R"( @@ -828,10 +806,8 @@ TEST_CASE("IntegratedSnarlFinder safely handles a single node cycle", "[genotype )"; // Make an actual graph - VG graph; - Graph chunk; - json2pb(chunk, graph_json.c_str(), graph_json.size()); - graph.merge(chunk); + bdsg::HashGraph graph; + vg::io::json2graph(graph_json, &graph); // Make an IntegratedSnarlFinder unique_ptr finder(new IntegratedSnarlFinder(graph)); @@ -844,7 +820,7 @@ TEST_CASE("IntegratedSnarlFinder safely handles a single node cycle", "[genotype } TEST_CASE("IntegratedSnarlFinder safely handles a totally connected graph", "[genotype][integrated-snarl-finder]") { - + // Build a toy graph const string graph_json = R"( @@ -866,10 +842,8 @@ TEST_CASE("IntegratedSnarlFinder safely handles a totally connected graph", "[ge )"; // Make an actual graph - VG graph; - Graph chunk; - json2pb(chunk, graph_json.c_str(), graph_json.size()); - graph.merge(chunk); + bdsg::HashGraph graph; + vg::io::json2graph(graph_json, &graph); // Make an IntegratedSnarlFinder unique_ptr finder(new IntegratedSnarlFinder(graph)); @@ -882,7 +856,7 @@ TEST_CASE("IntegratedSnarlFinder safely handles a totally connected graph", "[ge } TEST_CASE("IntegratedSnarlFinder prefers to root at a bridge edge path in a tie", "[genotype][integrated-snarl-finder]") { - + // Build a toy graph const string graph_json = R"( @@ -903,10 +877,8 @@ TEST_CASE("IntegratedSnarlFinder prefers to root at a bridge edge path in a tie" )"; // Make an actual graph - VG graph; - Graph chunk; - json2pb(chunk, graph_json.c_str(), graph_json.size()); - graph.merge(chunk); + bdsg::HashGraph graph; + vg::io::json2graph(graph_json, &graph); // Make an IntegratedSnarlFinder unique_ptr finder(new IntegratedSnarlFinder(graph)); @@ -935,7 +907,7 @@ TEST_CASE("IntegratedSnarlFinder prefers to root at a bridge edge path in a tie" } TEST_CASE("IntegratedSnarlFinder prefers to root at a cycle that is 1 bp longer", "[genotype][integrated-snarl-finder]") { - + // Build a toy graph const string graph_json = R"( @@ -956,10 +928,8 @@ TEST_CASE("IntegratedSnarlFinder prefers to root at a cycle that is 1 bp longer" )"; // Make an actual graph - VG graph; - Graph chunk; - json2pb(chunk, graph_json.c_str(), graph_json.size()); - graph.merge(chunk); + bdsg::HashGraph graph; + vg::io::json2graph(graph_json, &graph); // Make an IntegratedSnarlFinder unique_ptr finder(new IntegratedSnarlFinder(graph)); @@ -988,7 +958,7 @@ TEST_CASE("IntegratedSnarlFinder prefers to root at a cycle that is 1 bp longer" } TEST_CASE("IntegratedSnarlFinder prefers to root at a chain with an up-weighted node", "[genotype][integrated-snarl-finder]") { - + // Build a toy graph const string graph_json = R"( @@ -1009,10 +979,8 @@ TEST_CASE("IntegratedSnarlFinder prefers to root at a chain with an up-weighted )"; // Make an actual graph - VG graph; - Graph chunk; - json2pb(chunk, graph_json.c_str(), graph_json.size()); - graph.merge(chunk); + bdsg::HashGraph graph; + vg::io::json2graph(graph_json, &graph); // Make an IntegratedSnarlFinder that adds 10 bp to node 4's apparent length unique_ptr finder(new IntegratedSnarlFinder(graph, {{4, 10}})); @@ -1041,7 +1009,7 @@ TEST_CASE("IntegratedSnarlFinder prefers to root at a chain with an up-weighted } TEST_CASE("IntegratedSnarlFinder sees tips as disqualifying ultrabubbles", "[genotype][integrated-snarl-finder]") { - + // Build a toy graph const string graph_json = R"( @@ -1066,10 +1034,8 @@ TEST_CASE("IntegratedSnarlFinder sees tips as disqualifying ultrabubbles", "[gen )"; // Make an actual graph - VG graph; - Graph chunk; - json2pb(chunk, graph_json.c_str(), graph_json.size()); - graph.merge(chunk); + bdsg::HashGraph graph; + vg::io::json2graph(graph_json, &graph); // Make an IntegratedSnarlFinder unique_ptr finder(new IntegratedSnarlFinder(graph)); @@ -1098,10 +1064,10 @@ TEST_CASE("IntegratedSnarlFinder sees tips as disqualifying ultrabubbles", "[gen } TEST_CASE("CactusSnarlFinder throws an error instead of crashing when the graph has no edges", "[genotype][cactus-snarl-finder]") { - + // Build a toy graph const string graph_json = R"( - + { "node": [ {"id": 1, "sequence": "G"}, @@ -1115,14 +1081,12 @@ TEST_CASE("CactusSnarlFinder throws an error instead of crashing when the graph {"id": 9, "sequence": "A"} ] } - + )"; - + // Make an actual graph - VG graph; - Graph chunk; - json2pb(chunk, graph_json.c_str(), graph_json.size()); - graph.merge(chunk); + bdsg::HashGraph graph; + vg::io::json2graph(graph_json, &graph); // Make a CactusSnarlFinder unique_ptr finder(new CactusSnarlFinder(graph)); @@ -1183,7 +1147,7 @@ TEST_CASE("fixed priors can be assigned to genotypes", "[genotype]") { TEST_CASE("TrivialTraversalFinder can find traversals", "[genotype]") { // Build a toy graph const string graph_json = R"( - + { "node": [ {"id": 1, "sequence": "G"}, @@ -1208,7 +1172,7 @@ TEST_CASE("TrivialTraversalFinder can find traversals", "[genotype]") { {"from": 6, "to": 8}, {"from": 7, "to": 9}, {"from": 8, "to": 9} - + ], "path": [ {"name": "hint", "mapping": [ @@ -1219,14 +1183,12 @@ TEST_CASE("TrivialTraversalFinder can find traversals", "[genotype]") { ]} ] } - + )"; - + // Make an actual graph - VG graph; - Graph chunk; - json2pb(chunk, graph_json.c_str(), graph_json.size()); - graph.merge(chunk); + bdsg::HashGraph graph; + vg::io::json2graph(graph_json, &graph); // Make a site Snarl site; @@ -1329,12 +1291,10 @@ TEST_CASE("CactusSnarlFinder can differentiate ultrabubbles from snarls", "[geno ] } )"; - + // Make an actual graph - VG graph; - Graph chunk; - json2pb(chunk, graph_json.c_str(), graph_json.size()); - graph.merge(chunk); + bdsg::HashGraph graph; + vg::io::json2graph(graph_json, &graph); // Find the snarls CactusSnarlFinder cubs(graph); @@ -1381,10 +1341,8 @@ TEST_CASE("CactusSnarlFinder can differentiate ultrabubbles from snarls", "[geno )"; // Make an actual graph - VG graph; - Graph chunk; - json2pb(chunk, graph_json.c_str(), graph_json.size()); - graph.merge(chunk); + bdsg::HashGraph graph; + vg::io::json2graph(graph_json, &graph); // Find the snarls CactusSnarlFinder cubs(graph); @@ -1454,10 +1412,8 @@ TEST_CASE("IntegratedSnarlFinder can differentiate ultrabubbles from snarls", "[ )"; // Make an actual graph - VG graph; - Graph chunk; - json2pb(chunk, graph_json.c_str(), graph_json.size()); - graph.merge(chunk); + bdsg::HashGraph graph; + vg::io::json2graph(graph_json, &graph); // Find the snarls IntegratedSnarlFinder cubs(graph); @@ -1504,10 +1460,8 @@ TEST_CASE("IntegratedSnarlFinder can differentiate ultrabubbles from snarls", "[ )"; // Make an actual graph - VG graph; - Graph chunk; - json2pb(chunk, graph_json.c_str(), graph_json.size()); - graph.merge(chunk); + bdsg::HashGraph graph; + vg::io::json2graph(graph_json, &graph); // Find the snarls IntegratedSnarlFinder cubs(graph); @@ -1581,11 +1535,9 @@ TEST_CASE("RepresentativeTraversalFinder finds traversals correctly", "[genotype } )"; - // Make an actual graph + // Load the graph. Needs to be a vg because we will give it to a SupportAugmentedGraph later. VG graph; - Graph chunk; - json2pb(chunk, graph_json.c_str(), graph_json.size()); - graph.merge(chunk); + vg::io::json2graph(graph_json, &graph); // Find the snarls CactusSnarlFinder cubs(graph); @@ -1713,11 +1665,9 @@ TEST_CASE("RepresentativeTraversalFinder finds traversals of simple inversions", } )"; - // Make an actual graph + // Load the graph. Needs to be a vg because we will give it to a SupportAugmentedGraph later. VG graph; - Graph chunk; - json2pb(chunk, graph_json.c_str(), graph_json.size()); - graph.merge(chunk); + vg::io::json2graph(graph_json, &graph); // Find the snarls CactusSnarlFinder cubs(graph); @@ -1774,11 +1724,11 @@ TEST_CASE("GBWTTraversalFinder finds traversals for GBWT threads", "[genotype][g string graph_json = R"({"node": [{"id": 1, "sequence": "CAAATAAGGCTT"}, {"id": 2, "sequence": "G"}, {"id": 3, "sequence": "GGAAATTTTC"}, {"id": 4, "sequence": "C"}, {"id": 5, "sequence": "TGGAGTTCTATTATATTCC"}, {"id": 6, "sequence": "G"}, {"id": 7, "sequence": "A"}, {"id": 8, "sequence": "ACTCTCTGGTTCCTG"}, {"id": 9, "sequence": "A"}, {"id": 10, "sequence": "G"}, {"id": 11, "sequence": "TGCTATGTGTAACTAGTAATGGTAATGGATATGTTGGGCTTTTTTCTTTGATTTATTTGAAGTGACGTTTGACAATCTATCACTAGGGGTAATGTGGGGAAATGGAAAGAATACAAGATTTGGAGCCA"}], "edge": [{"from": 1, "to": 2}, {"from": 1, "to": 3}, {"from": 2, "to": 3}, {"from": 3, "to": 4}, {"from": 3, "to": 5}, {"from": 4, "to": 5}, {"from": 5, "to": 6}, {"from": 5, "to": 7}, {"from": 6, "to": 8}, {"from": 7, "to": 8}, {"from": 8, "to": 9}, {"from": 8, "to": 10}, {"from": 9, "to": 11}, {"from": 10, "to": 11}]})"; // Load the JSON - vg::Graph proto_graph; - json2pb(proto_graph, graph_json.c_str(), graph_json.size()); + bdsg::HashGraph graph; + vg::io::json2graph(graph_json, &graph); // Build the xg index xg::XG xg_index; - xg_index.from_path_handle_graph(vg::VG(proto_graph)); + xg_index.from_path_handle_graph(graph); gbwt::Verbosity::set(gbwt::Verbosity::SILENT); diff --git a/src/unittest/genotyper.cpp b/src/unittest/genotyper.cpp index e2e9f7a142..4228b16ee3 100644 --- a/src/unittest/genotyper.cpp +++ b/src/unittest/genotyper.cpp @@ -7,6 +7,7 @@ #include "../snarls.hpp" #include "../cactus_snarl_finder.hpp" #include "../traversal_finder.hpp" +#include "../io/json2graph.hpp" namespace vg { namespace unittest { @@ -41,15 +42,6 @@ TEST_CASE("traversals can be found from reads", "[genotyper]") { {"from": 6, "to": 8}, {"from": 7, "to": 9}, {"from": 8, "to": 9} - - ], - "path": [ - {"name": "hint", "mapping": [ - {"position": {"node_id": 1}, "rank" : 1 }, - {"position": {"node_id": 6}, "rank" : 2 }, - {"position": {"node_id": 8}, "rank" : 3 }, - {"position": {"node_id": 9}, "rank" : 4 } - ]} ] } @@ -57,9 +49,7 @@ TEST_CASE("traversals can be found from reads", "[genotyper]") { // Make an actual graph VG graph; - Graph chunk; - json2pb(chunk, graph_json.c_str(), graph_json.size()); - graph.merge(chunk); + vg::io::json2graph(graph_json, &graph); // Find the snarls SnarlManager manager = CactusSnarlFinder(graph).find_snarls(); diff --git a/src/unittest/haplotypes.cpp b/src/unittest/haplotypes.cpp index e441bbe197..9e4e04475b 100644 --- a/src/unittest/haplotypes.cpp +++ b/src/unittest/haplotypes.cpp @@ -4,8 +4,10 @@ #include "catch.hpp" #include "haplotypes.hpp" +#include "../io/json2graph.hpp" #include "xg.hpp" #include "vg.hpp" +#include #include @@ -66,7 +68,7 @@ TEST_CASE("We can represent appropriate graphs according to linear reference", " )"; thread_t SNP_thread = {tm[1], tm[3], tm[4]}; - + string del_graph_json = R"( {"node":[ {"id":1,"sequence":"AAA"}, @@ -89,22 +91,24 @@ TEST_CASE("We can represent appropriate graphs according to linear reference", " ]} ]} )"; - + thread_t del_ref_thread = {tm[1], tm[2], tm[4]}; thread_t del_thread = {tm[1], tm[4]}; - - vg::Graph SNP_proto_graph; - json2pb(SNP_proto_graph, SNP_graph_json.c_str(), SNP_graph_json.size()); + + // Build the SNP graph + bdsg::HashGraph SNP_graph; + vg::io::json2graph(SNP_graph_json, &SNP_graph); // Build the xg index xg::XG SNP_xg_index; - SNP_xg_index.from_path_handle_graph(vg::VG(SNP_proto_graph)); + SNP_xg_index.from_path_handle_graph(SNP_graph); vg::path_handle_t SNP_ref_path_handle = SNP_xg_index.get_path_handle("reference"); - - vg::Graph del_proto_graph; - json2pb(del_proto_graph, del_graph_json.c_str(), del_graph_json.size()); + + // Build the del graph + bdsg::HashGraph del_graph; + vg::io::json2graph(del_graph_json, &del_graph); // Build the xg index xg::XG del_xg_index; - del_xg_index.from_path_handle_graph(vg::VG(del_proto_graph)); + del_xg_index.from_path_handle_graph(del_graph); vg::path_handle_t del_ref_path_handle = del_xg_index.get_path_handle("reference"); // NEGATIVE SNVs @@ -159,18 +163,20 @@ TEST_CASE("We can represent appropriate graphs according to linear reference", " thread_t double_thread = {tm[1], tm[2], tm[4]}; - vg::Graph long_proto_graph; - json2pb(long_proto_graph, long_graph_json.c_str(), long_graph_json.size()); + // Build the long graph + bdsg::HashGraph long_graph; + vg::io::json2graph(long_graph_json, &long_graph); // Build the xg index xg::XG long_xg_index; - long_xg_index.from_path_handle_graph(vg::VG(long_proto_graph)); + long_xg_index.from_path_handle_graph(long_graph); vg::path_handle_t long_ref_path_handle = long_xg_index.get_path_handle("reference"); - - vg::Graph double_proto_graph; - json2pb(double_proto_graph, double_graph_json.c_str(), double_graph_json.size()); + + // Build the double graph + bdsg::HashGraph double_graph; + vg::io::json2graph(double_graph_json, &double_graph); // Build the xg index xg::XG double_xg_index; - double_xg_index.from_path_handle_graph(vg::VG(double_proto_graph)); + double_xg_index.from_path_handle_graph(double_graph); vg::path_handle_t double_ref_path_handle = double_xg_index.get_path_handle("reference"); string matching_test_file = "matching_test.slls"; @@ -382,13 +388,13 @@ TEST_CASE("We can score haplotypes using GBWT", "[haplo-score][gbwt]") { TEST_CASE("We can recognize a required crossover", "[hapo-score][gbwt]") { // This graph is the start of xy2 from test/small string graph_json = R"({"node": [{"id": 1, "sequence": "CAAATAAGGCTT"}, {"id": 2, "sequence": "G"}, {"id": 3, "sequence": "GGAAATTTTC"}, {"id": 4, "sequence": "C"}, {"id": 5, "sequence": "TGGAGTTCTATTATATTCC"}, {"id": 6, "sequence": "G"}, {"id": 7, "sequence": "A"}, {"id": 8, "sequence": "ACTCTCTGGTTCCTG"}, {"id": 9, "sequence": "A"}, {"id": 10, "sequence": "G"}, {"id": 11, "sequence": "TGCTATGTGTAACTAGTAATGGTAATGGATATGTTGGGCTTTTTTCTTTGATTTATTTGAAGTGACGTTTGACAATCTATCACTAGGGGTAATGTGGGGAAATGGAAAGAATACAAGATTTGGAGCCA"}], "edge": [{"from": 1, "to": 2}, {"from": 1, "to": 3}, {"from": 2, "to": 3}, {"from": 3, "to": 4}, {"from": 3, "to": 5}, {"from": 4, "to": 5}, {"from": 5, "to": 6}, {"from": 5, "to": 7}, {"from": 6, "to": 8}, {"from": 7, "to": 8}, {"from": 8, "to": 9}, {"from": 8, "to": 10}, {"from": 9, "to": 11}, {"from": 10, "to": 11}]})"; - - // Load the JSON - vg::Graph proto_graph; - json2pb(proto_graph, graph_json.c_str(), graph_json.size()); + + // Load the JSON into a HashGraph + bdsg::HashGraph graph; + vg::io::json2graph(graph_json, &graph); // Build the xg index xg::XG xg_index; - xg_index.from_path_handle_graph(vg::VG(proto_graph)); + xg_index.from_path_handle_graph(graph); gbwt::Verbosity::set(gbwt::Verbosity::SILENT); gbwt::DynamicGBWT* gbwt_index = new gbwt::DynamicGBWT; diff --git a/src/unittest/indexed_vg.cpp b/src/unittest/indexed_vg.cpp index 7f74d92193..27504dea9f 100644 --- a/src/unittest/indexed_vg.cpp +++ b/src/unittest/indexed_vg.cpp @@ -40,7 +40,7 @@ TEST_CASE("An IndexedVG can be created for a single node", "[handle][indexed-vg] ] })"; - // Load the JSON + // Load the JSON to Protobuf specifically. Graph proto_graph; json2pb(proto_graph, graph_json.c_str(), graph_json.size()); diff --git a/src/unittest/mapper.cpp b/src/unittest/mapper.cpp index 2caf42d076..17f81fe17b 100644 --- a/src/unittest/mapper.cpp +++ b/src/unittest/mapper.cpp @@ -1,9 +1,10 @@ /// \file mapper.cpp -/// +/// /// unit tests for the mapper #include #include "vg/io/json2pb.h" +#include "../io/json2graph.hpp" #include #include #include "../mapper.hpp" @@ -25,14 +26,10 @@ TEST_CASE( "Mapper can map to a one-node graph", "[mapping][mapper]" ) { ]} ] })"; - + // Load the JSON - Graph proto_graph; - json2pb(proto_graph, graph_json.c_str(), graph_json.size()); - - // Make it into a VG - VG graph; - graph.extend(proto_graph); + bdsg::HashGraph graph; + vg::io::json2graph(graph_json, &graph); // Make GCSA quiet gcsa::Verbosity::set(gcsa::Verbosity::SILENT); @@ -245,14 +242,10 @@ TEST_CASE( "Mapper finds optimal mapping for read starting with node-border MEM" {"position":{"node_id":1444},"rank":1059}, {"position":{"node_id":1445},"rank":1060}]}]} )"; - + // Load the JSON - Graph proto_graph; - json2pb(proto_graph, graph_json.c_str(), graph_json.size()); - - // Make it into a VG - VG graph; - graph.extend(proto_graph); + bdsg::HashGraph graph; + vg::io::json2graph(graph_json, &graph); // Make GCSA quiet gcsa::Verbosity::set(gcsa::Verbosity::SILENT); @@ -311,14 +304,10 @@ TEST_CASE( "Mapper can annotate positions correctly on both strands", "[mapper][ ]} ]} )"; - + // Load the JSON - Graph proto_graph; - json2pb(proto_graph, graph_json.c_str(), graph_json.size()); - - // Make it into a VG - VG graph; - graph.extend(proto_graph); + bdsg::HashGraph graph; + vg::io::json2graph(graph_json, &graph); // Make GCSA quiet gcsa::Verbosity::set(gcsa::Verbosity::SILENT); diff --git a/src/unittest/minimizer_mapper.cpp b/src/unittest/minimizer_mapper.cpp index e2e25db870..cb6f0bf7d3 100644 --- a/src/unittest/minimizer_mapper.cpp +++ b/src/unittest/minimizer_mapper.cpp @@ -3,8 +3,8 @@ /// unit tests for the minimizer mapper #include -#include "vg/io/json2pb.h" #include "../io/json2graph.hpp" +#include #include #include "../minimizer_mapper.hpp" #include "../build_index.hpp" @@ -450,15 +450,13 @@ TEST_CASE("MinimizerMapper can map an empty string between odd points", "[giraff {"id": "55511925", "sequence": "CTTCCTTCC"} ] })"; - - // TODO: Write a json_to_handle_graph - vg::Graph proto_graph; - json2pb(proto_graph, graph_json.c_str(), graph_json.size()); - auto graph = vg::VG(proto_graph); - + + bdsg::HashGraph graph; + vg::io::json2graph(graph_json, &graph); + Alignment aln; aln.set_sequence(""); - + pos_t left_anchor {55511921, false, 5}; // This is on the final base of the node pos_t right_anchor {55511925, false, 6}; @@ -480,7 +478,7 @@ TEST_CASE("MinimizerMapper can map an empty string between odd points", "[giraff TEST_CASE("MinimizerMapper can map with an initial deletion", "[giraffe][mapping][right_tail]") { Aligner aligner; - + string graph_json = R"({ "edge": [ {"from": "1", "to": "2"}, @@ -492,12 +490,10 @@ TEST_CASE("MinimizerMapper can map with an initial deletion", "[giraffe][mapping {"id": "3", "sequence": "CATTAG"} ] })"; - - // TODO: Write a json_to_handle_graph - vg::Graph proto_graph; - json2pb(proto_graph, graph_json.c_str(), graph_json.size()); - auto graph = vg::VG(proto_graph); - + + bdsg::HashGraph graph; + vg::io::json2graph(graph_json, &graph); + Alignment aln; aln.set_sequence("CATTAG"); @@ -527,7 +523,7 @@ TEST_CASE("MinimizerMapper can map with an initial deletion", "[giraffe][mapping TEST_CASE("MinimizerMapper can map with an initial deletion on a multi-base node", "[giraffe][mapping][right_tail]") { Aligner aligner; - + string graph_json = R"({ "edge": [ {"from": "1", "to": "2"}, @@ -539,12 +535,10 @@ TEST_CASE("MinimizerMapper can map with an initial deletion on a multi-base node {"id": "3", "sequence": "CATTAG"} ] })"; - - // TODO: Write a json_to_handle_graph - vg::Graph proto_graph; - json2pb(proto_graph, graph_json.c_str(), graph_json.size()); - auto graph = vg::VG(proto_graph); - + + bdsg::HashGraph graph; + vg::io::json2graph(graph_json, &graph); + Alignment aln; aln.set_sequence("CATTAG"); @@ -574,7 +568,7 @@ TEST_CASE("MinimizerMapper can map with an initial deletion on a multi-base node TEST_CASE("MinimizerMapper can map right off the past-the-end base", "[giraffe][mapping][right_tail]") { Aligner aligner; - + string graph_json = R"({ "edge": [ {"from": "1", "to": "2"}, @@ -586,15 +580,13 @@ TEST_CASE("MinimizerMapper can map right off the past-the-end base", "[giraffe][ {"id": "3", "sequence": "CATTAG"} ] })"; - - // TODO: Write a json_to_handle_graph - vg::Graph proto_graph; - json2pb(proto_graph, graph_json.c_str(), graph_json.size()); - auto graph = vg::VG(proto_graph); - + + bdsg::HashGraph graph; + vg::io::json2graph(graph_json, &graph); + Alignment aln; aln.set_sequence("CATTAG"); - + pos_t left_anchor {1, false, 1}; // This is the past-end position pos_t right_anchor = empty_pos_t(); @@ -635,15 +627,13 @@ TEST_CASE("MinimizerMapper can compute longest detectable gap in range", "[giraf TEST_CASE("MinimizerMapper can find a significant indel instead of a tempting softclip", "[giraffe][mapping][left_tail]") { Aligner aligner; - + string graph_json = R"({ "edge": [{"from": "30788083", "to": "30788088"}, {"from": "30788083", "to": "30788084"}, {"from": "30788074", "to": "30788075"}, {"from": "30788074", "to": "30788076"}, {"from": "30788079", "to": "30788080"}, {"from": "30788079", "to": "30788081"}, {"from": "30788086", "to": "30788088"}, {"from": "30788086", "to": "30788087", "to_end": true}, {"from": "30788075", "to": "30788077"}, {"from": "30788073", "to": "30788074"}, {"from": "30788078", "to": "30788079"}, {"from": "30788077", "to": "30788078"}, {"from": "30788084", "to": "30788088"}, {"from": "30788084", "to": "30788085"}, {"from": "30788076", "to": "30788077"}, {"from": "30788087", "from_start": true, "to": "30788088"}, {"from": "30788081", "to": "30788082"}, {"from": "30788080", "to": "30788082"}, {"from": "30788082", "to": "30788088"}, {"from": "30788082", "to": "30788083"}, {"from": "30788085", "to": "30788086"}], "node": [{"id": "30788083", "sequence": "AAA"}, {"id": "30788074", "sequence": "AAAAAAAATACAAAAAATTAGC"}, {"id": "30788079", "sequence": "CGCCACTGCACTCCAGCCTGGGC"}, {"id": "30788086", "sequence": "AAAAAAA"}, {"id": "30788075", "sequence": "T"}, {"id": "30788073", "sequence": "GAAAGAGAGTTGTTTAAATTCCATAGTTAGGGCCGGGCGCGGTGGCTCACGCCTGTAATCCCAGCACTTTGGGAGGCCGAGGCGGGCGGATCACGAGGTCAGGAGATCGAGACCATCCTGGCTAACACGGTGAAACCCCGTCTCTACTA"}, {"id": "30788078", "sequence": "G"}, {"id": "30788077", "sequence": "GGGCGTGGTAGCGGGCGCCTGTAGTCCCAGCTACTCGGGAGGCTGAGGCAGGAGAATGGCGTGAACCCGGGAGGCGGAGCTTGCAGTGAGCCGAGATC"}, {"id": "30788084", "sequence": "A"}, {"id": "30788088", "sequence": "AATTCCATAGTTAGAAAAATAAGACATATCAGGTTTTCAAAAAGTGTAGCCATTTTCTGTTTCTAAAAGGGACACTTAAAGTGAAA"}, {"id": "30788076", "sequence": "C"}, {"id": "30788087", "sequence": "T"}, {"id": "30788081", "sequence": "A"}, {"id": "30788080", "sequence": "G"}, {"id": "30788082", "sequence": "ACAGAGCGAGACTCCGTCTCAAAAAAAAAAAAAA"}, {"id": "30788085", "sequence": "AA"}] })"; - - // TODO: Write a json_to_handle_graph - vg::Graph proto_graph; - json2pb(proto_graph, graph_json.c_str(), graph_json.size()); - auto graph = vg::VG(proto_graph); + + bdsg::HashGraph graph; + vg::io::json2graph(graph_json, &graph); Alignment aln; aln.set_sequence("TTGAAAACCTGATATGTCTTATTTTTCTAACTATGGAATTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTGAGACGGAGTCTCGCTCTGTCGCCCAGGCTGGAGTGCAGTGGCGCGATCTCGGCTCACTGCAAGCTCCGCCTCCCGGGTTCACGCCATTCTCCTGCCTCAGCCTCCCGAGTAGCTGGGACTACAGGCGCCCGCTACCACGCCCGGCTAATTTTTTGTATTTTTTTT"); @@ -854,9 +844,8 @@ TEST_CASE("MinimizerMapper can extract a strand-split dagified local graph witho {"id": "60245278", "sequence": "GATTACAGATTACA"}] } )"; - vg::Graph graph_chunk; - json2pb(graph_chunk, graph_json.c_str(), graph_json.size()); - vg::VG graph(graph_chunk); + bdsg::HashGraph graph; + vg::io::json2graph(graph_json, &graph); TestMinimizerMapper::with_dagified_local_graph(make_pos_t(60245283, false, 10), empty_pos_t(), 50, graph, [&](DeletableHandleGraph& dagified_graph, const handle_t& left_anchor_handle, const handle_t& right_anchor_handle, const std::function(const handle_t&)>& dagified_handle_to_base) { // The graph started as a stick diff --git a/src/unittest/multipath_alignment_graph.cpp b/src/unittest/multipath_alignment_graph.cpp index bea5f687aa..d78e19d6f1 100644 --- a/src/unittest/multipath_alignment_graph.cpp +++ b/src/unittest/multipath_alignment_graph.cpp @@ -3,7 +3,8 @@ /// unit tests for the multipath mapper's MultipathAlignmentGraph #include -#include "vg/io/json2pb.h" +#include "../io/json2graph.hpp" +#include #include #include "../cactus_snarl_finder.hpp" #include "../integrated_snarl_finder.hpp" @@ -47,13 +48,9 @@ TEST_CASE( "MultipathAlignmentGraph::align handles tails correctly", "[multipath })"; // Load the JSON - Graph proto_graph; - json2pb(proto_graph, graph_json.c_str(), graph_json.size()); - - // Make it into a VG - VG vg; - vg.extend(proto_graph); - + bdsg::HashGraph vg; + ::vg::io::json2graph(graph_json, &vg); + // Make snarls on it CactusSnarlFinder bubble_finder(vg); IntegratedSnarlFinder snarl_finder(vg); diff --git a/src/unittest/multipath_mapper.cpp b/src/unittest/multipath_mapper.cpp index be6d3b6194..bc1dc4cdd9 100644 --- a/src/unittest/multipath_mapper.cpp +++ b/src/unittest/multipath_mapper.cpp @@ -4,7 +4,9 @@ #include #include "vg/io/json2pb.h" +#include "../io/json2graph.hpp" #include +#include #include "../multipath_mapper.hpp" #include "../build_index.hpp" #include "xg.hpp" @@ -111,7 +113,7 @@ TEST_CASE( "MultipathMapper::read_coverage works", "[multipath][mapping][multipa } TEST_CASE( "MultipathMapper::query_cluster_graphs works", "[multipath][mapping][multipathmapper]" ) { - + string graph_json = R"({ "node": [{"id": 1, "sequence": "GATTACA"}], "path": [ @@ -120,14 +122,10 @@ TEST_CASE( "MultipathMapper::query_cluster_graphs works", "[multipath][mapping][ ]} ] })"; - - // Load the JSON - Graph proto_graph; - json2pb(proto_graph, graph_json.c_str(), graph_json.size()); - - // Make it into a VG - VG graph; - graph.extend(proto_graph); + + // Load the JSON into a HashGraph + bdsg::HashGraph graph; + vg::io::json2graph(graph_json, &graph); // Make GCSA quiet gcsa::Verbosity::set(gcsa::Verbosity::SILENT); @@ -135,17 +133,17 @@ TEST_CASE( "MultipathMapper::query_cluster_graphs works", "[multipath][mapping][ // Make pointers to fill in gcsa::GCSA* gcsaidx = nullptr; gcsa::LCPArray* lcpidx = nullptr; - + // Build the GCSA index build_gcsa_lcp(graph, gcsaidx, lcpidx, 16, 3); - + // Build the xg index xg::XG xg_index; - xg_index.from_path_handle_graph(VG(proto_graph)); - + xg_index.from_path_handle_graph(graph); + // Make a multipath mapper to map against the graph. TestMultipathMapper mapper(&xg_index, gcsaidx, lcpidx); - + // Make an Alignment that we're pretending we're doing Alignment aln; aln.set_sequence("GATTACA"); @@ -264,7 +262,7 @@ TEST_CASE( "MultipathMapper::query_cluster_graphs works", "[multipath][mapping][ } TEST_CASE( "MultipathMapper can map to a one-node graph", "[multipath][mapping][multipathmapper]" ) { - + string graph_json = R"({ "node": [{"id": 1, "sequence": "GATTACA"}], "path": [ @@ -273,14 +271,10 @@ TEST_CASE( "MultipathMapper can map to a one-node graph", "[multipath][mapping][ ]} ] })"; - - // Load the JSON - Graph proto_graph; - json2pb(proto_graph, graph_json.c_str(), graph_json.size()); - - // Make it into a VG - VG graph; - graph.extend(proto_graph); + + // Load the JSON into a HashGraph + bdsg::HashGraph graph; + vg::io::json2graph(graph_json, &graph); // Make GCSA quiet gcsa::Verbosity::set(gcsa::Verbosity::SILENT); @@ -291,11 +285,11 @@ TEST_CASE( "MultipathMapper can map to a one-node graph", "[multipath][mapping][ // Build the GCSA index build_gcsa_lcp(graph, gcsaidx, lcpidx, 16, 3); - + // Build the xg index xg::XG xg_index; xg_index.from_path_handle_graph(graph); - + // Make a multipath mapper to map against the graph. MultipathMapper mapper(&xg_index, gcsaidx, lcpidx); // Lower the max mapping quality so that it thinks it can find unambiguous mappings of @@ -422,16 +416,12 @@ TEST_CASE( "MultipathMapper can map to a one-node graph", "[multipath][mapping][ } TEST_CASE( "MultipathMapper can work on a bigger graph", "[multipath][mapping][multipathmapper]" ) { - + string graph_json = R"({"node":[{"sequence":"CTTCTCATCCCTCCTCAAGGGCCTTTAACTACTCCACATCCAAAGCTACCCAGGCCATTTTAAGTTTCCTGTGGACTAAGGACAAAGGTGCGGGGAGATG","id":12},{"sequence":"A","id":2},{"sequence":"CAAATAAGGCTTGGAAATTTTCTGGAGTTCTATTATATTCCAACTCTCTGGTTCCTGGTGCTATGTGTAACTAGTAATGGTAATGGATATGTTGGGCTTT","id":3},{"sequence":"TTTCTTTGATTTATTTGAAGTGACGTTTGACAATCTATCACTAGGGGTAATGTGGGGAAATGGAAAGAATACAAGATTTGGAGCCAGACAAATCTGGGTT","id":4},{"sequence":"CAAATCCTCACTTTGCCACATATTAGCCATGTGACTTTGAACAAGTTAGTTAATCTCTCTGAACTTCAGTTTAATTATCTCTAATATGGAGATGATACTA","id":5},{"sequence":"CTGACAGCAGAGGTTTGCTGTGAAGATTAAATTAGGTGATGCTTGTAAAGCTCAGGGAATAGTGCCTGGCATAGAGGAAAGCCTCTGACAACTGGTAGTT","id":6},{"sequence":"ACTGTTATTTACTATGAATCCTCACCTTCCTTGACTTCTTGAAACATTTGGCTATTGACCTCTTTCCTCCTTGAGGCTCTTCTGGCTTTTCATTGTCAAC","id":7},{"sequence":"ACAGTCAACGCTCAATACAAGGGACATTAGGATTGGCAGTAGCTCAGAGATCTCTCTGCTCACCGTGATCTTCAAGTTTGAAAATTGCATCTCAAATCTA","id":8},{"sequence":"AGACCCAGAGGGCTCACCCAGAGTCGAGGCTCAAGGACAGCTCTCCTTTGTGTCCAGAGTGTATACGATGTAACTCTGTTCGGGCACTGGTGAAAGATAA","id":9},{"sequence":"CAGAGGAAATGCCTGGCTTTTTATCAGAACATGTTTCCAAGCTTATCCCTTTTCCCAGCTCTCCTTGTCCCTCCCAAGATCTCTTCACTGGCCTCTTATC","id":10},{"sequence":"TTTACTGTTACCAAATCTTTCCAGAAGCTGCTCTTTCCCTCAATTGTTCATTTGTCTTCTTGTCCAGGAATGAACCACTGCTCTCTTCTTGTCAGATCAG","id":11}],"path":[{"name":"x","mapping":[{"position":{"node_id":3},"edit":[{"from_length":100,"to_length":100}],"rank":1},{"position":{"node_id":4},"edit":[{"from_length":100,"to_length":100}],"rank":2},{"position":{"node_id":5},"edit":[{"from_length":100,"to_length":100}],"rank":3},{"position":{"node_id":6},"edit":[{"from_length":100,"to_length":100}],"rank":4},{"position":{"node_id":7},"edit":[{"from_length":100,"to_length":100}],"rank":5},{"position":{"node_id":8},"edit":[{"from_length":100,"to_length":100}],"rank":6},{"position":{"node_id":9},"edit":[{"from_length":100,"to_length":100}],"rank":7},{"position":{"node_id":10},"edit":[{"from_length":100,"to_length":100}],"rank":8},{"position":{"node_id":11},"edit":[{"from_length":100,"to_length":100}],"rank":9},{"position":{"node_id":12},"edit":[{"from_length":100,"to_length":100}],"rank":10},{"position":{"node_id":2},"edit":[{"from_length":1,"to_length":1}],"rank":11}]}],"edge":[{"from":12,"to":2},{"from":3,"to":4},{"from":4,"to":5},{"from":5,"to":6},{"from":6,"to":7},{"from":7,"to":8},{"from":8,"to":9},{"from":9,"to":10},{"from":10,"to":11},{"from":11,"to":12}]})"; - - // Load the JSON - Graph proto_graph; - json2pb(proto_graph, graph_json.c_str(), graph_json.size()); - - // Make it into a VG - VG graph; - graph.extend(proto_graph); + + // Load the JSON into a HashGraph + bdsg::HashGraph graph; + vg::io::json2graph(graph_json, &graph); // Make GCSA quiet gcsa::Verbosity::set(gcsa::Verbosity::SILENT); @@ -442,11 +432,11 @@ TEST_CASE( "MultipathMapper can work on a bigger graph", "[multipath][mapping][m // Build the GCSA index build_gcsa_lcp(graph, gcsaidx, lcpidx, 16, 3); - + // Build the xg index xg::XG xg_index; - xg_index.from_path_handle_graph(VG(proto_graph)); - + xg_index.from_path_handle_graph(graph); + // Make a multipath mapper to map against the graph. TestMultipathMapper mapper(&xg_index, gcsaidx, lcpidx); // Lower the max mapping quality so that it thinks it can find unambiguous mappings of diff --git a/src/unittest/path_component_index.cpp b/src/unittest/path_component_index.cpp index 058f4bf9c1..edd3a6013a 100644 --- a/src/unittest/path_component_index.cpp +++ b/src/unittest/path_component_index.cpp @@ -8,7 +8,8 @@ #include "path_component_index.hpp" #include "xg.hpp" #include "vg.hpp" -#include "vg/io/json2pb.h" +#include "../io/json2graph.hpp" +#include #include namespace vg { @@ -17,14 +18,14 @@ namespace unittest { TEST_CASE("Path component memoization produces expected results", "[pathcomponent]") { string graph_json = R"({"node": [{"sequence": "AAACCC", "id": 1}, {"sequence": "CACACA", "id": 2}, {"sequence": "CACACA", "id": 3}, {"sequence": "TTTTGG", "id": 4}, {"sequence": "ACGTAC", "id": 5}], "path": [{"name": "one", "mapping": [{"position": {"node_id": 1}, "rank": 1}, {"position": {"node_id": 2}, "rank": 2}]}, {"name": "three", "mapping": [{"position": {"node_id": 2}, "rank": 1}, {"position": {"node_id": 3}, "rank": 2}]}, {"name": "two", "mapping": [{"position": {"node_id": 4}, "rank": 1}, {"position": {"node_id": 5}, "rank": 2}]}], "edge": [{"from": 1, "to": 2}, {"from": 2, "to": 3}, {"from": 4, "to": 5}]})"; - + // Load the JSON - Graph proto_graph; - json2pb(proto_graph, graph_json.c_str(), graph_json.size()); - + bdsg::HashGraph graph; + vg::io::json2graph(graph_json, &graph); + // Build the xg index xg::XG xg_index; - xg_index.from_path_handle_graph(VG(proto_graph)); + xg_index.from_path_handle_graph(graph); unordered_set comp_1; diff --git a/src/unittest/path_index.cpp b/src/unittest/path_index.cpp index b70152ae2d..e1facc2977 100644 --- a/src/unittest/path_index.cpp +++ b/src/unittest/path_index.cpp @@ -5,9 +5,9 @@ #include #include -#include "vg/io/json2pb.h" -#include +#include "../io/json2graph.hpp" #include "../path_index.hpp" +#include #include "catch.hpp" namespace vg { @@ -58,15 +58,11 @@ const string path_index_graph_1 = R"( TEST_CASE("PathIndex can be created", "[pathindex]") { - + // Load the graph - Graph graph; - json2pb(graph, path_index_graph_1.c_str(), path_index_graph_1.size()); - - // Make it into a VG - VG to_index; - to_index.extend(graph); - + bdsg::HashGraph to_index; + vg::io::json2graph(path_index_graph_1, &to_index); + // Make a PathIndex PathIndex index(to_index, "cool", true); @@ -78,13 +74,9 @@ TEST_CASE("PathIndex can be created", "[pathindex]") { TEST_CASE("PathIndex translation can change a node ID", "[pathindex]") { // Load the graph - Graph graph; - json2pb(graph, path_index_graph_1.c_str(), path_index_graph_1.size()); - - // Make it into a VG - VG to_index; - to_index.extend(graph); - + bdsg::HashGraph to_index; + vg::io::json2graph(path_index_graph_1, &to_index); + // Make a PathIndex PathIndex index(to_index, "cool", true); @@ -115,15 +107,11 @@ TEST_CASE("PathIndex translation can change a node ID", "[pathindex]") { } TEST_CASE("PathIndex translation can divide a node", "[pathindex]") { - + // Load the graph - Graph graph; - json2pb(graph, path_index_graph_1.c_str(), path_index_graph_1.size()); - - // Make it into a VG - VG to_index; - to_index.extend(graph); - + bdsg::HashGraph to_index; + vg::io::json2graph(path_index_graph_1, &to_index); + // Make a PathIndex PathIndex index(to_index, "cool", true); @@ -174,15 +162,11 @@ TEST_CASE("PathIndex translation can divide a node", "[pathindex]") { } TEST_CASE("PathIndex translation can create reverse strand mappings", "[pathindex]") { - + // Load the graph - Graph graph; - json2pb(graph, path_index_graph_1.c_str(), path_index_graph_1.size()); - - // Make it into a VG - VG to_index; - to_index.extend(graph); - + bdsg::HashGraph to_index; + vg::io::json2graph(path_index_graph_1, &to_index); + // Make a PathIndex PathIndex index(to_index, "cool", true); @@ -235,15 +219,11 @@ TEST_CASE("PathIndex translation can create reverse strand mappings", "[pathinde } TEST_CASE("PathIndex translation can handle translations articulated for the reverse strand", "[pathindex]") { - + // Load the graph - Graph graph; - json2pb(graph, path_index_graph_1.c_str(), path_index_graph_1.size()); - - // Make it into a VG - VG to_index; - to_index.extend(graph); - + bdsg::HashGraph to_index; + vg::io::json2graph(path_index_graph_1, &to_index); + // Make a PathIndex PathIndex index(to_index, "cool", true); @@ -300,15 +280,11 @@ TEST_CASE("PathIndex translation can handle translations articulated for the rev } TEST_CASE("PathIndex translation can divide the last node", "[pathindex]") { - + // Load the graph - Graph graph; - json2pb(graph, path_index_graph_1.c_str(), path_index_graph_1.size()); - - // Make it into a VG - VG to_index; - to_index.extend(graph); - + bdsg::HashGraph to_index; + vg::io::json2graph(path_index_graph_1, &to_index); + // Make a PathIndex PathIndex index(to_index, "cool", true); diff --git a/src/unittest/phase_unfolder.cpp b/src/unittest/phase_unfolder.cpp index 0c79972941..36cfbca9de 100644 --- a/src/unittest/phase_unfolder.cpp +++ b/src/unittest/phase_unfolder.cpp @@ -12,7 +12,8 @@ #include #include "../phase_unfolder.hpp" -#include "vg/io/json2pb.h" +#include "../io/json2graph.hpp" +#include #include "xg.hpp" #include "catch.hpp" @@ -210,10 +211,10 @@ const std::string unfolder_graph_path = R"( TEST_CASE("PhaseUnfolder can unfold XG paths", "[phaseunfolder][indexing]") { // Build an XG index with a path. - Graph graph_with_path; - json2pb(graph_with_path, unfolder_graph_path.c_str(), unfolder_graph_path.size()); + bdsg::HashGraph graph_with_path; + vg::io::json2graph(unfolder_graph_path, &graph_with_path); xg::XG xg_index; - xg_index.from_path_handle_graph(VG(graph_with_path)); + xg_index.from_path_handle_graph(graph_with_path); // Build an empty GBWT index. gbwt::GBWT gbwt_index; @@ -224,9 +225,7 @@ TEST_CASE("PhaseUnfolder can unfold XG paths", "[phaseunfolder][indexing]") { // Build a VG graph. VG vg_graph; - Graph temp_graph; - json2pb(temp_graph, unfolder_graph.c_str(), unfolder_graph.size()); - vg_graph.merge(temp_graph); + vg::io::json2graph(unfolder_graph, &vg_graph); // Remove branching regions from the VG graph, including the last node, // but keep the edge (1, 6) in the graph. @@ -255,10 +254,10 @@ TEST_CASE("PhaseUnfolder can unfold XG paths", "[phaseunfolder][indexing]") { TEST_CASE("PhaseUnfolder can restore XG paths", "[phaseunfolder][indexing]") { // Build an XG index with a path. - Graph graph_with_path; - json2pb(graph_with_path, unfolder_graph_path.c_str(), unfolder_graph_path.size()); + bdsg::HashGraph graph_with_path; + vg::io::json2graph(unfolder_graph_path, &graph_with_path); xg::XG xg_index; - xg_index.from_path_handle_graph(VG(graph_with_path)); + xg_index.from_path_handle_graph(graph_with_path); // Build an empty GBWT index. gbwt::GBWT gbwt_index; @@ -269,9 +268,7 @@ TEST_CASE("PhaseUnfolder can restore XG paths", "[phaseunfolder][indexing]") { // Build a VG graph. VG vg_graph; - Graph temp_graph; - json2pb(temp_graph, unfolder_graph.c_str(), unfolder_graph.size()); - vg_graph.merge(temp_graph); + vg::io::json2graph(unfolder_graph, &vg_graph); // Remove branching regions from the VG graph, including the last node, // but keep the edge (1, 6) in the graph. @@ -299,10 +296,10 @@ TEST_CASE("PhaseUnfolder can restore XG paths", "[phaseunfolder][indexing]") { TEST_CASE("PhaseUnfolder can unfold GBWT threads", "[phaseunfolder][indexing]") { // Build an XG index without a path. - Graph graph_without_path; - json2pb(graph_without_path, unfolder_graph.c_str(), unfolder_graph.size()); + bdsg::HashGraph graph_without_path; + vg::io::json2graph(unfolder_graph, &graph_without_path); xg::XG xg_index; - xg_index.from_path_handle_graph(VG(graph_without_path)); + xg_index.from_path_handle_graph(graph_without_path); // Build a GBWT with three threads including a duplicate. We want to have // only one instance of short_path unfolded, but we want separate copies @@ -335,9 +332,7 @@ TEST_CASE("PhaseUnfolder can unfold GBWT threads", "[phaseunfolder][indexing]") // Build a VG graph. VG vg_graph; - Graph temp_graph; - json2pb(temp_graph, unfolder_graph.c_str(), unfolder_graph.size()); - vg_graph.merge(temp_graph); + vg::io::json2graph(unfolder_graph, &vg_graph); // Remove branching regions from the VG graph, including the last node, // but keep the edge (1, 6) in the graph. @@ -366,10 +361,10 @@ TEST_CASE("PhaseUnfolder can unfold GBWT threads", "[phaseunfolder][indexing]") TEST_CASE("PhaseUnfolder can unfold both XG paths and GBWT threads", "[phaseunfolder][indexing]") { // Build an XG index with a path. - Graph graph_with_path; - json2pb(graph_with_path, unfolder_graph_path.c_str(), unfolder_graph_path.size()); + bdsg::HashGraph graph_with_path; + vg::io::json2graph(unfolder_graph_path, &graph_with_path); xg::XG xg_index; - xg_index.from_path_handle_graph(VG(graph_with_path)); + xg_index.from_path_handle_graph(graph_with_path); // Build a GBWT with three threads including a duplicate. We want to have // only one instance of short_path unfolded, but we want separate copies @@ -402,9 +397,7 @@ TEST_CASE("PhaseUnfolder can unfold both XG paths and GBWT threads", "[phaseunfo // Build a VG graph. VG vg_graph; - Graph temp_graph; - json2pb(temp_graph, unfolder_graph.c_str(), unfolder_graph.size()); - vg_graph.merge(temp_graph); + vg::io::json2graph(unfolder_graph, &vg_graph); // Remove branching regions from the VG graph, including the last node, // but keep the edge (1, 6) in the graph. @@ -501,10 +494,10 @@ const std::string unfolder_graph_simple_path = R"( TEST_CASE("PhaseUnfolder can merge shared prefixes and suffixes", "[phaseunfolder][indexing]") { // Build an XG index. - Graph simple_graph; - json2pb(simple_graph, unfolder_graph_simple.c_str(), unfolder_graph_simple.size()); + bdsg::HashGraph simple_graph; + vg::io::json2graph(unfolder_graph_simple, &simple_graph); xg::XG xg_index; - xg_index.from_path_handle_graph(VG(simple_graph)); + xg_index.from_path_handle_graph(simple_graph); // Build a GBWT with both possible threads. gbwt::vector_type upper_path { @@ -536,9 +529,7 @@ TEST_CASE("PhaseUnfolder can merge shared prefixes and suffixes", "[phaseunfolde // Build a VG graph. VG vg_graph; - Graph temp_graph; - json2pb(temp_graph, unfolder_graph_simple.c_str(), unfolder_graph_simple.size()); - vg_graph.merge(temp_graph); + vg::io::json2graph(unfolder_graph_simple, &vg_graph); // Remove the bubble, including its endpoints. std::set to_remove { 3, 4, 5, 6 }; @@ -566,10 +557,10 @@ TEST_CASE("PhaseUnfolder can merge shared prefixes and suffixes", "[phaseunfolde TEST_CASE("PhaseUnfolder can extend short threads", "[phaseunfolder][indexing]") { // Build an XG index. - Graph simple_graph_with_path; - json2pb(simple_graph_with_path, unfolder_graph_simple_path.c_str(), unfolder_graph_simple_path.size()); + bdsg::HashGraph simple_graph_with_path; + vg::io::json2graph(unfolder_graph_simple_path, &simple_graph_with_path); xg::XG xg_index; - xg_index.from_path_handle_graph(VG(simple_graph_with_path)); + xg_index.from_path_handle_graph(simple_graph_with_path); // Build a GBWT for the fragment that is different from the reference. gbwt::vector_type short_fragment { @@ -586,9 +577,7 @@ TEST_CASE("PhaseUnfolder can extend short threads", "[phaseunfolder][indexing]") // Build a VG graph. VG vg_graph; - Graph temp_graph; - json2pb(temp_graph, unfolder_graph_simple.c_str(), unfolder_graph_simple.size()); - vg_graph.merge(temp_graph); + vg::io::json2graph(unfolder_graph_simple, &vg_graph); // Remove the bubble, including its endpoints. std::set to_remove { 3, 4, 5, 6 }; diff --git a/src/unittest/readfilter.cpp b/src/unittest/readfilter.cpp index cc1562f3f3..6d84fa0a38 100644 --- a/src/unittest/readfilter.cpp +++ b/src/unittest/readfilter.cpp @@ -5,6 +5,9 @@ #include "catch.hpp" #include "readfilter.hpp" #include "xg.hpp" +#include "vg/io/json2pb.h" +#include "../io/json2graph.hpp" +#include namespace vg { namespace unittest { @@ -44,13 +47,13 @@ TEST_CASE("reads with ambiguous ends can be trimmed", "[filter]") { )"; - // Load it into Protobuf - Graph chunk; - json2pb(chunk, graph_json.c_str(), graph_json.size()); - + // Load the graph + bdsg::HashGraph chunk; + vg::io::json2graph(graph_json, &chunk); + // Pass it over to XG xg::XG index; - index.from_path_handle_graph(VG(chunk)); + index.from_path_handle_graph(chunk); // Make a ReadFilter; ReadFilter filter; diff --git a/src/unittest/sampler.cpp b/src/unittest/sampler.cpp index d8bb95b650..cda0147f57 100644 --- a/src/unittest/sampler.cpp +++ b/src/unittest/sampler.cpp @@ -6,11 +6,10 @@ #include #include -#include "vg/io/json2pb.h" -#include +#include "../io/json2graph.hpp" +#include #include "../sampler.hpp" #include "../xg.hpp" -#include "../vg.hpp" #include "catch.hpp" namespace vg { @@ -28,13 +27,9 @@ TEST_CASE( "Sampler can sample from a 1-node graph", "[sampler]" ) { })"; // Load the JSON - Graph proto_graph; - json2pb(proto_graph, graph_json.c_str(), graph_json.size()); - - // Make it into a VG - VG graph; - graph.extend(proto_graph); - + bdsg::HashGraph graph; + vg::io::json2graph(graph_json, &graph); + // Build the xg index xg::XG xg_index; xg_index.from_path_handle_graph(graph); @@ -118,13 +113,9 @@ TEST_CASE( "position_at works", "[sampler]" ) { })"; // Load the JSON - Graph proto_graph; - json2pb(proto_graph, graph_json.c_str(), graph_json.size()); - - // Make it into a VG - VG graph; - graph.extend(proto_graph); - + bdsg::HashGraph graph; + vg::io::json2graph(graph_json, &graph); + // Build the xg index xg::XG xg_index; xg_index.from_path_handle_graph(graph); @@ -195,13 +186,9 @@ TEST_CASE( "Sampler can sample from a loop-containing path", "[sampler]" ) { })"; // Load the JSON - Graph proto_graph; - json2pb(proto_graph, graph_json.c_str(), graph_json.size()); - - // Make it into a VG - VG graph; - graph.extend(proto_graph); - + bdsg::HashGraph graph; + vg::io::json2graph(graph_json, &graph); + // Build the xg index xg::XG xg_index; xg_index.from_path_handle_graph(graph); @@ -259,13 +246,9 @@ TEST_CASE( "Sampler can across reversing edges", "[sampler]" ) { })"; // Load the JSON - Graph proto_graph; - json2pb(proto_graph, graph_json.c_str(), graph_json.size()); - - // Make it into a VG - VG graph; - graph.extend(proto_graph); - + bdsg::HashGraph graph; + vg::io::json2graph(graph_json, &graph); + // Build the xg index xg::XG xg_index; xg_index.from_path_handle_graph(graph); diff --git a/src/unittest/snarl_distance_index.cpp b/src/unittest/snarl_distance_index.cpp index 093c45d0aa..0c308a00b0 100644 --- a/src/unittest/snarl_distance_index.cpp +++ b/src/unittest/snarl_distance_index.cpp @@ -9,8 +9,8 @@ #include #include #include -#include "vg/io/json2pb.h" -#include +#include "../io/json2graph.hpp" +#include #include "catch.hpp" #include "support/random_graph.hpp" #include "support/randomness.hpp" @@ -3446,12 +3446,9 @@ namespace vg { // } // )"; // - // VG graph; - // // // Load up the graph - // Graph g; - // json2pb(g, graph_json.c_str(), graph_json.size()); - // graph.extend(g); + // VG graph; + // vg::io::json2graph(graph_json, &graph); // // // Define the one snarl // Snarl snarl1; @@ -3578,12 +3575,9 @@ namespace vg { // string snarl2_json = R"({"type": 1, "end": {"node_id": 187209, "backward": true}, "start": {"node_id": 178895, "backward": true}, "parent": {"end": {"node_id": 187208}, "start": {"node_id": 178894}}})"; // string snarl3_json = R"({"type": 1, "end": {"node_id": 178896}, "start": {"node_id": 178895}, "parent": {"end": {"node_id": 187208}, "start": {"node_id": 178894}}})"; // - // VG graph; - // // // Load up the graph - // Graph g; - // json2pb(g, graph_json.c_str(), graph_json.size()); - // graph.extend(g); + // VG graph; + // vg::io::json2graph(graph_json, &graph); // // // Load the snarls // Snarl snarl1, snarl2, snarl3; @@ -3754,9 +3748,7 @@ namespace vg { // Make an actual graph VG graph; - Graph chunk; - json2pb(chunk, graph_json.c_str(), graph_json.size()); - graph.extend(chunk); + vg::io::json2graph(graph_json, &graph); // We need to see the path. REQUIRE(graph.paths.size() == 1); @@ -4014,9 +4006,7 @@ namespace vg { // Make an actual graph VG graph; - Graph chunk; - json2pb(chunk, graph_json.c_str(), graph_json.size()); - graph.extend(chunk); + vg::io::json2graph(graph_json, &graph); IntegratedSnarlFinder snarl_finder(graph); SnarlDistanceIndex distance_index; @@ -4127,9 +4117,7 @@ namespace vg { // Make an actual graph VG graph; - Graph chunk; - json2pb(chunk, graph_json.c_str(), graph_json.size()); - graph.extend(chunk); + vg::io::json2graph(graph_json, &graph); IntegratedSnarlFinder snarl_finder(graph); SnarlDistanceIndex distance_index; @@ -4276,9 +4264,7 @@ namespace vg { // Make an actual graph VG graph; - Graph chunk; - json2pb(chunk, graph_json.c_str(), graph_json.size()); - graph.extend(chunk); + vg::io::json2graph(graph_json, &graph); IntegratedSnarlFinder snarl_finder(graph); SnarlDistanceIndex distance_index; @@ -4405,9 +4391,7 @@ namespace vg { // Make an actual graph VG graph; - Graph chunk; - json2pb(chunk, graph_json.c_str(), graph_json.size()); - graph.extend(chunk); + vg::io::json2graph(graph_json, &graph); IntegratedSnarlFinder snarl_finder(graph); SnarlDistanceIndex distance_index; @@ -4514,9 +4498,7 @@ namespace vg { // Make an actual graph VG graph; - Graph chunk; - json2pb(chunk, graph_json.c_str(), graph_json.size()); - graph.extend(chunk); + vg::io::json2graph(graph_json, &graph); IntegratedSnarlFinder snarl_finder(graph); SnarlDistanceIndex distance_index; @@ -4618,9 +4600,7 @@ namespace vg { // Make an actual graph VG graph; - Graph chunk; - json2pb(chunk, graph_json.c_str(), graph_json.size()); - graph.extend(chunk); + vg::io::json2graph(graph_json, &graph); IntegratedSnarlFinder snarl_finder(graph); SnarlDistanceIndex distance_index; @@ -4788,9 +4768,7 @@ namespace vg { // Make an actual graph VG graph; - Graph chunk; - json2pb(chunk, graph_json.c_str(), graph_json.size()); - graph.extend(chunk); + vg::io::json2graph(graph_json, &graph); IntegratedSnarlFinder snarl_finder(graph); SnarlDistanceIndex distance_index; @@ -4911,9 +4889,7 @@ namespace vg { // Make an actual graph VG graph; - Graph chunk; - json2pb(chunk, graph_json.c_str(), graph_json.size()); - graph.extend(chunk); + vg::io::json2graph(graph_json, &graph); IntegratedSnarlFinder snarl_finder(graph); SnarlDistanceIndex distance_index; diff --git a/src/unittest/snarls.cpp b/src/unittest/snarls.cpp index c2f5030326..c7edf85b05 100644 --- a/src/unittest/snarls.cpp +++ b/src/unittest/snarls.cpp @@ -9,6 +9,8 @@ #include #include #include "vg/io/json2pb.h" +#include "../io/json2graph.hpp" +#include #include #include "catch.hpp" #include "support/random_graph.hpp" @@ -1697,14 +1699,12 @@ namespace vg { ] } )"; - + VG graph; - + // Load up the graph - Graph g; - json2pb(g, graph_json.c_str(), graph_json.size()); - graph.extend(g); - + vg::io::json2graph(graph_json, &graph); + // Define the one snarl Snarl snarl1; snarl1.mutable_start()->set_node_id(6462830); @@ -1830,14 +1830,12 @@ namespace vg { string snarl1_json = R"({"type": 1, "end": {"node_id": 187208}, "start": {"node_id": 178894}})"; string snarl2_json = R"({"type": 1, "end": {"node_id": 187209, "backward": true}, "start": {"node_id": 178895, "backward": true}, "parent": {"end": {"node_id": 187208}, "start": {"node_id": 178894}}})"; string snarl3_json = R"({"type": 1, "end": {"node_id": 178896}, "start": {"node_id": 178895}, "parent": {"end": {"node_id": 187208}, "start": {"node_id": 178894}}})"; - + VG graph; - + // Load up the graph - Graph g; - json2pb(g, graph_json.c_str(), graph_json.size()); - graph.extend(g); - + vg::io::json2graph(graph_json, &graph); + // Load the snarls Snarl snarl1, snarl2, snarl3; json2pb(snarl1, snarl1_json.c_str(), snarl1_json.size()); @@ -1917,13 +1915,11 @@ namespace vg { } )"; - + // Make an actual graph VG graph; - Graph chunk; - json2pb(chunk, graph_json.c_str(), graph_json.size()); - graph.extend(chunk); - + vg::io::json2graph(graph_json, &graph); + // We need to see the path. REQUIRE(graph.paths.size() == 1); @@ -2045,10 +2041,8 @@ namespace vg { // Make an actual graph VG graph; - Graph chunk; - json2pb(chunk, graph_json.c_str(), graph_json.size()); - graph.extend(chunk); - + vg::io::json2graph(graph_json, &graph); + SnarlManager snarl_manager = CactusSnarlFinder(graph).find_snarls(); #ifdef debug @@ -2061,7 +2055,7 @@ namespace vg { cerr << endl; }); #endif - + SECTION("Root node has 1 child bubble") { REQUIRE(snarl_manager.top_level_snarls().size() == 1); @@ -2127,15 +2121,13 @@ namespace vg { ]} ] } - + )"; - + // Make an actual graph VG graph; - Graph chunk; - json2pb(chunk, graph_json.c_str(), graph_json.size()); - graph.extend(chunk); - + vg::io::json2graph(graph_json, &graph); + SnarlManager snarl_manager = CactusSnarlFinder(graph).find_snarls(); #ifdef debug @@ -2246,15 +2238,13 @@ namespace vg { ]} ] } - + )"; - + // Make an actual graph VG graph; - Graph chunk; - json2pb(chunk, graph_json.c_str(), graph_json.size()); - graph.extend(chunk); - + vg::io::json2graph(graph_json, &graph); + SnarlManager snarl_manager = CactusSnarlFinder(graph).find_snarls(); #ifdef debug @@ -2354,18 +2344,16 @@ namespace vg { {"from": 2, "to": 4}, {"from": 2, "to": 3}, {"from": 2, "to": 2}, - {"from": 3, "to": 3} + {"from": 3, "to": 3} ] } - + )"; - + // Make an actual graph VG graph; - Graph chunk; - json2pb(chunk, graph_json.c_str(), graph_json.size()); - graph.extend(chunk); - + vg::io::json2graph(graph_json, &graph); + SnarlManager snarl_manager = CactusSnarlFinder(graph).find_snarls(); #ifdef debug @@ -2415,15 +2403,13 @@ namespace vg { ]} ] } - + )"; - + // Make an actual graph VG graph; - Graph chunk; - json2pb(chunk, graph_json.c_str(), graph_json.size()); - graph.extend(chunk); - + vg::io::json2graph(graph_json, &graph); + SnarlManager snarl_manager = CactusSnarlFinder(graph).find_snarls(); #ifdef debug @@ -2490,18 +2476,16 @@ namespace vg { "edge": [ {"from": 1, "to": 2}, {"from": 2, "to": 1} - + ] } - + )"; - + // Make an actual graph VG graph; - Graph chunk; - json2pb(chunk, graph_json.c_str(), graph_json.size()); - graph.extend(chunk); - + vg::io::json2graph(graph_json, &graph); + SnarlManager snarl_manager = CactusSnarlFinder(graph).find_snarls(); #ifdef debug @@ -2555,15 +2539,13 @@ namespace vg { {"from": 3, "to": 6} ] } - + )"; - + // Make an actual graph VG graph; - Graph chunk; - json2pb(chunk, graph_json.c_str(), graph_json.size()); - graph.extend(chunk); - + vg::io::json2graph(graph_json, &graph); + SnarlManager snarl_manager = CactusSnarlFinder(graph).find_snarls(); #ifdef debug @@ -2767,15 +2749,13 @@ namespace vg { {"from": 9, "to": 10} ] } - + )"; - + // Make an actual graph VG graph; - Graph chunk; - json2pb(chunk, graph_json.c_str(), graph_json.size()); - graph.extend(chunk); - + vg::io::json2graph(graph_json, &graph); + SnarlManager snarl_manager = CactusSnarlFinder(graph).find_snarls(); #ifdef debug snarl_manager.for_each_snarl_preorder([&](const Snarl* snarl) { @@ -3919,14 +3899,12 @@ namespace vg { {"position": {"node_id": 7, "is_reverse" : "true"}, "rank" : 5 } ]} ] - } + } )"; - + // Make an actual graph VG graph; - Graph chunk; - json2pb(chunk, graph_json.c_str(), graph_json.size()); - graph.extend(chunk); + vg::io::json2graph(graph_json, &graph); assert(graph.is_valid()); SECTION( "PathTraversalFinder can find simple forward traversals") { diff --git a/src/unittest/source_sink_overlay.cpp b/src/unittest/source_sink_overlay.cpp index 4c0ecbc20f..bf2aa3bc13 100644 --- a/src/unittest/source_sink_overlay.cpp +++ b/src/unittest/source_sink_overlay.cpp @@ -10,7 +10,8 @@ #include "../source_sink_overlay.hpp" #include "../kmer.hpp" #include "../vg.hpp" -#include "vg/io/json2pb.h" +#include "../io/json2graph.hpp" +#include #include #include @@ -132,11 +133,9 @@ TEST_CASE("SourceSinkOverlay adds a source and a sink to a 1-node graph", "[over TEST_CASE("SourceSinkOverlay agrees with VG::add_start_end_markers in a tiny graph", "[overlay]") { const string graph_json = R"({"node":[{"sequence":"CAAATAAG","id":"1"},{"sequence":"A","id":"2"},{"sequence":"G","id":"3"},{"sequence":"T","id":"4"},{"sequence":"C","id":"5"},{"sequence":"TTG","id":"6"},{"sequence":"A","id":"7"},{"sequence":"G","id":"8"},{"sequence":"AAATTTTCTGGAGTTCTAT","id":"9"},{"sequence":"A","id":"10"},{"sequence":"T","id":"11"},{"sequence":"ATAT","id":"12"},{"sequence":"A","id":"13"},{"sequence":"T","id":"14"},{"sequence":"CCAACTCTCTG","id":"15"}],"edge":[{"from":"1","to":"2"},{"from":"1","to":"3"},{"from":"2","to":"4"},{"from":"2","to":"5"},{"from":"3","to":"4"},{"from":"3","to":"5"},{"from":"4","to":"6"},{"from":"5","to":"6"},{"from":"6","to":"7"},{"from":"6","to":"8"},{"from":"7","to":"9"},{"from":"8","to":"9"},{"from":"9","to":"10"},{"from":"9","to":"11"},{"from":"10","to":"12"},{"from":"11","to":"12"},{"from":"12","to":"13"},{"from":"12","to":"14"},{"from":"13","to":"15"},{"from":"14","to":"15"}],"path":[{"name":"x","mapping":[{"position":{"node_id":"1"},"edit":[{"from_length":8,"to_length":8}],"rank":"1"},{"position":{"node_id":"3"},"edit":[{"from_length":1,"to_length":1}],"rank":"2"},{"position":{"node_id":"5"},"edit":[{"from_length":1,"to_length":1}],"rank":"3"},{"position":{"node_id":"6"},"edit":[{"from_length":3,"to_length":3}],"rank":"4"},{"position":{"node_id":"8"},"edit":[{"from_length":1,"to_length":1}],"rank":"5"},{"position":{"node_id":"9"},"edit":[{"from_length":19,"to_length":19}],"rank":"6"},{"position":{"node_id":"11"},"edit":[{"from_length":1,"to_length":1}],"rank":"7"},{"position":{"node_id":"12"},"edit":[{"from_length":4,"to_length":4}],"rank":"8"},{"position":{"node_id":"14"},"edit":[{"from_length":1,"to_length":1}],"rank":"9"},{"position":{"node_id":"15"},"edit":[{"from_length":11,"to_length":11}],"rank":"10"}]}]})"; - - Graph graph; - json2pb(graph, graph_json); - - VG produced(graph); + + VG produced; + vg::io::json2graph(graph_json, &produced); id_t highest_id = produced.max_node_id(); id_t start_id = highest_id + 1; diff --git a/src/unittest/variant_adder.cpp b/src/unittest/variant_adder.cpp index afe3353e4b..6fad7d82ab 100644 --- a/src/unittest/variant_adder.cpp +++ b/src/unittest/variant_adder.cpp @@ -9,7 +9,7 @@ #include "../utility.hpp" #include "../path.hpp" -#include "vg/io/json2pb.h" +#include "../io/json2graph.hpp" #include #include @@ -38,7 +38,7 @@ ref 5 rs1337 A G 29 PASS . GT // Make a stream out of the data std::stringstream vcf_stream(vcf_data); - + // Load it up in vcflib vcflib::VariantCallFile vcf; vcf.open(vcf_stream); @@ -51,14 +51,10 @@ ref 5 rs1337 A G 29 PASS . GT ]} ] })"; - + // Load the JSON - Graph proto_graph; - json2pb(proto_graph, graph_json.c_str(), graph_json.size()); - - // Make it into a VG VG graph; - graph.extend(proto_graph); + json2graph(graph_json, &graph); // Make a VariantAdder @@ -85,7 +81,7 @@ ref 5 rs1337 A G 29 PASS . GT 0/1 // Make a stream out of the data std::stringstream vcf_stream(vcf_data); - + // Load it up in vcflib vcflib::VariantCallFile vcf; vcf.open(vcf_stream); @@ -98,14 +94,10 @@ ref 5 rs1337 A G 29 PASS . GT 0/1 ]} ] })"; - + // Load the JSON - Graph proto_graph; - json2pb(proto_graph, graph_json.c_str(), graph_json.size()); - - // Make it into a VG VG graph; - graph.extend(proto_graph); + json2graph(graph_json, &graph); // Make a VariantAdder VariantAdder adder(graph); @@ -139,7 +131,7 @@ ref 5 rs1337 AAAAAAAAAAAAAAAAAAAAA A 29 PASS . GT 0/1 // Make a stream out of the data std::stringstream vcf_stream(vcf_data); - + // Load it up in vcflib vcflib::VariantCallFile vcf; vcf.open(vcf_stream); @@ -152,14 +144,10 @@ ref 5 rs1337 AAAAAAAAAAAAAAAAAAAAA A 29 PASS . GT 0/1 ]} ] })"; - + // Load the JSON - Graph proto_graph; - json2pb(proto_graph, graph_json.c_str(), graph_json.size()); - - // Make it into a VG VG graph; - graph.extend(proto_graph); + json2graph(graph_json, &graph); // Make a VariantAdder VariantAdder adder(graph); @@ -193,7 +181,7 @@ ref 5 rs1337 AAAAAAAAAAAAAAAAAAAAA A 29 PASS . GT 0/1 // Make a stream out of the data std::stringstream vcf_stream(vcf_data); - + // Load it up in vcflib vcflib::VariantCallFile vcf; vcf.open(vcf_stream); @@ -213,14 +201,10 @@ ref 5 rs1337 AAAAAAAAAAAAAAAAAAAAA A 29 PASS . GT 0/1 ]} ] })"; - + // Load the JSON - Graph proto_graph; - json2pb(proto_graph, graph_json.c_str(), graph_json.size()); - - // Make it into a VG VG graph; - graph.extend(proto_graph); + json2graph(graph_json, &graph); SECTION ("should work when the graph is as given") { @@ -280,7 +264,7 @@ ref 5 rs1337 AAAAAAAAAAAAAAAAAAAAA AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA 29 // Make a stream out of the data std::stringstream vcf_stream(vcf_data); - + // Load it up in vcflib vcflib::VariantCallFile vcf; vcf.open(vcf_stream); @@ -293,14 +277,10 @@ ref 5 rs1337 AAAAAAAAAAAAAAAAAAAAA AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA 29 ]} ] })"; - + // Load the JSON - Graph proto_graph; - json2pb(proto_graph, graph_json.c_str(), graph_json.size()); - - // Make it into a VG VG graph; - graph.extend(proto_graph); + json2graph(graph_json, &graph); // Make a VariantAdder VariantAdder adder(graph); @@ -323,14 +303,10 @@ TEST_CASE( "The smart aligner works on very large inserts", "[variantadder]" ) { string graph_json = R"({ "node": [{"id": 1, "sequence": "GCGCAAAAAAAAAAAAAAAAAAAAAGCGC"}] })"; - + // Load the JSON - Graph proto_graph; - json2pb(proto_graph, graph_json.c_str(), graph_json.size()); - - // Make it into a VG VG graph; - graph.extend(proto_graph); + json2graph(graph_json, &graph); // Make a VariantAdder VariantAdder adder(graph); @@ -396,21 +372,17 @@ TEST_CASE( "The smart aligner should use mapping offsets on huge deletions", "[v {"from": 2, "to": 3} ] })"; - + // Make the graph have lots of As stringstream a_stream; for(size_t i = 0; i < 10000; i++) { a_stream << "A"; } graph_json = regex_replace(graph_json, std::regex("<10kAs>"), a_stream.str()); - + // Load the JSON - Graph proto_graph; - json2pb(proto_graph, graph_json.c_str(), graph_json.size()); - - // Make it into a VG VG graph; - graph.extend(proto_graph); + json2graph(graph_json, &graph); // Make a VariantAdder VariantAdder adder(graph); @@ -484,21 +456,17 @@ TEST_CASE( "The smart aligner should find existing huge deletions", "[variantadd {"from": 2, "to": 3} ] })"; - + // Make the graph have lots of As stringstream a_stream; for(size_t i = 0; i < 10000; i++) { a_stream << "A"; } graph_json = regex_replace(graph_json, std::regex("<10kAs>"), a_stream.str()); - + // Load the JSON - Graph proto_graph; - json2pb(proto_graph, graph_json.c_str(), graph_json.size()); - - // Make it into a VG VG graph; - graph.extend(proto_graph); + json2graph(graph_json, &graph); // Make a VariantAdder VariantAdder adder(graph); @@ -564,21 +532,17 @@ TEST_CASE( "The smart aligner should use deletion edits on medium deletions", "[ string graph_json = R"({ "node": [{"id": 1, "sequence": "GCGC<100As>GCGC"}] })"; - + // Make the graph have lots of As stringstream a_stream; for(size_t i = 0; i < 100; i++) { a_stream << "A"; } graph_json = regex_replace(graph_json, std::regex("<100As>"), a_stream.str()); - + // Load the JSON - Graph proto_graph; - json2pb(proto_graph, graph_json.c_str(), graph_json.size()); - - // Make it into a VG VG graph; - graph.extend(proto_graph); + json2graph(graph_json, &graph); // Make a VariantAdder VariantAdder adder(graph); diff --git a/src/unittest/vg_algorithms.cpp b/src/unittest/vg_algorithms.cpp index b4fc736734..8e713f87f7 100644 --- a/src/unittest/vg_algorithms.cpp +++ b/src/unittest/vg_algorithms.cpp @@ -27,7 +27,7 @@ #include "../vg.hpp" #include "../xg.hpp" #include -#include "vg/io/json2pb.h" +#include "../io/json2graph.hpp" using namespace google::protobuf; @@ -1092,11 +1092,8 @@ TEST_CASE( "Connecting graph extraction works on a cool loop without leaving ext {"edge": [{"from": "185927720", "to": "185927722"}, {"from": "185927721", "from_start": true, "to": "185927722"}, {"from": "185927722", "to": "186681786", "to_end": true}, {"from": "185927722", "to": "185927723"}, {"from": "186681786", "to": "186683083"}, {"from": "186681786", "from_start": true, "to": "186681787", "to_end": true}, {"from": "186681787", "to": "186683069", "to_end": true}, {"from": "186681787", "from_start": true, "to": "186681789"}, {"from": "186681787", "from_start": true, "to": "186681788", "to_end": true}, {"from": "186681788", "from_start": true, "to": "186681790", "to_end": true}, {"from": "186681789", "to": "186681790", "to_end": true}, {"from": "186681790", "from_start": true, "to": "186681792", "to_end": true}, {"from": "186683069", "from_start": true, "to": "186683079", "to_end": true}, {"from": "186683079", "from_start": true, "to": "186683080", "to_end": true}, {"from": "186683080", "from_start": true, "to": "186683081", "to_end": true}, {"from": "186683081", "from_start": true, "to": "186683083", "to_end": true}], "node": [{"id": "185927720", "sequence": "G"}, {"id": "185927721", "sequence": "A"}, {"id": "185927722", "sequence": "ACCGGG"}, {"id": "185927723", "sequence": "AGTGGGGG"}, {"id": "186681786", "sequence": "C"}, {"id": "186681787", "sequence": "TGGGAGTCTAAGTCTCTTTTGATCACACTTTAAAGACCAAAAGGTAGAAGCGCAAAGACGTTATCTGTCCAATATTACAAACCTAGTAAGTGGTGGAATTTGGCCTTGAACCCAGATCTGTAACTCCAGAGCCGAAGTGCTTCACCCACCTCCCTGTGGTG"}, {"id": "186681788", "sequence": "G"}, {"id": "186681789", "sequence": "T"}, {"id": "186681790", "sequence": "TAT"}, {"id": "186681792", "sequence": "T"}, {"id": "186683069", "sequence": "G"}, {"id": "186683079", "sequence": "G"}, {"id": "186683080", "sequence": "TACCCCGGAATCCCTGCCGCGGCCCCTCGGGCCTGTCCACATCCCTCTGCCCCTCCCAGACCTCTGTCCTTCCACCAATCGCCTCCCGCAGCCCCGAGCCGCCACTCCCAGTCCCCCGAGTCCCTGCCGCGCGCCCTCGCGCCTGTCCACATCCCTCTGCCCATCCGAGACCTCTGTCCTTACACCACTAGCCACCCCACGTGGGACTTCCATGGCTTCTGAGTACAAGGCCAGCCCCCCGGCCCACCAGCTTTCGGAATGCCTGCTTACCTCTTTTTCTGTAGA"}, {"id": "186683081", "sequence": "CCGG"}, {"id": "186683083", "sequence": "C"}]} )"; - Graph source; - json2pb(source, graph_json.c_str(), graph_json.size()); - VG vg; - vg.extend(source); + vg::io::json2graph(graph_json, &vg); bdsg::HashGraph extractor; @@ -1688,11 +1685,8 @@ TEST_CASE( "Connecting graph extraction works on a particular case without leavi )"; - Graph source; - json2pb(source, graph_json.c_str(), graph_json.size()); - VG vg; - vg.extend(source); + vg::io::json2graph(graph_json, &vg); VG extractor; @@ -2583,13 +2577,9 @@ TEST_CASE( "Topological sort works on a more complex graph", {"node": [{"id": 1, "sequence": "GTATTTTTAGTA"}, {"id": 2, "sequence": "G"}, {"id": 3, "sequence": "GAGACGGGGTTTCACCATGTT"}, {"id": 4, "sequence": "T"}, {"id": 5, "sequence": "CTAATTTTT"}, {"id": 6, "sequence": "CA"}, {"id": 7, "sequence": "GG"}, {"id": 8, "sequence": "ACGCCC"}, {"id": 9, "sequence": "C"}, {"id": 10, "sequence": "T"}, {"id": 11, "sequence": "C"}, {"id": 12, "sequence": "GCCA"}, {"id": 13, "sequence": "A"}, {"id": 14, "sequence": "GGGATTACAGGCGCACACC"}, {"id": 15, "sequence": "CCACACC"}, {"id": 16, "sequence": "AT"}, {"id": 17, "sequence": "CC"}, {"id": 18, "sequence": "GGTCAGGCTGGTCTCGACTCC"}, {"id": 19, "sequence": "TGACCTCCTGATCTGCCCCCC"}, {"id": 20, "sequence": "A"}, {"id": 21, "sequence": "G"}, {"id": 22, "sequence": "TATTTTTAGTA"}, {"id": 23, "sequence": "A"}, {"id": 24, "sequence": "G"}, {"id": 25, "sequence": "GA"}], "edge": [{"from": 4, "to": 1}, {"from": 5, "to": 1}, {"from": 1, "to": 2}, {"from": 1, "to": 3}, {"from": 22, "to": 2}, {"from": 2, "to": 20}, {"from": 2, "to": 21}, {"from": 3, "to": 18}, {"from": 5, "to": 4}, {"from": 6, "to": 5}, {"from": 7, "to": 5}, {"from": 8, "to": 6}, {"from": 8, "to": 7}, {"from": 9, "to": 8}, {"from": 10, "to": 8}, {"from": 11, "to": 9}, {"from": 11, "to": 10}, {"from": 12, "to": 11}, {"from": 13, "to": 11}, {"from": 16, "to": 12}, {"from": 17, "to": 12}, {"from": 12, "to": 15}, {"from": 14, "to": 13}, {"from": 18, "to": 19}, {"from": 20, "to": 25}, {"from": 21, "to": 25}, {"from": 23, "to": 22}, {"from": 24, "to": 22}]} )"; - // Load the JSON - Graph proto_graph; - json2pb(proto_graph, graph_json.c_str(), graph_json.size()); - - // Make it into a VG + // Load the JSON into a VG VG vg; - vg.extend(proto_graph); + vg::io::json2graph(graph_json, &vg); SECTION( "handlealgs::topological_order produces a consistent total ordering and orientation" ) { auto handle_sort = handlealgs::topological_order(&vg); @@ -5385,11 +5375,8 @@ TEST_CASE("simplify_siblings() works on a graph with a reversing self loop", "[a {"edge": [{"from": "1", "to": "3"}, {"from": "1", "to": "2"}, {"from": "2", "to": "2", "to_end": true}], "node": [{"id": "1", "sequence": "T"}, {"id": "2", "sequence": "A"}, {"id": "3", "sequence": "ACA"}], "path": [{"mapping": [{"edit": [{"from_length": 1, "to_length": 1}], "position": {"node_id": "1"}, "rank": "1"}, {"edit": [{"from_length": 1, "to_length": 1}], "position": {"node_id": "2"}, "rank": "2"}, {"edit": [{"from_length": 1, "to_length": 1}], "position": {"is_reverse": true, "node_id": "2"}, "rank": "3"}], "name": "x"}, {"mapping": [{"edit": [{"from_length": 1, "to_length": 1}], "position": {"node_id": "1"}, "rank": "1"}, {"edit": [{"from_length": 3, "to_length": 3}], "position": {"node_id": "3"}, "rank": "2"}], "name": "y"}]} )"; - Graph source; - json2pb(source, graph_json.c_str(), graph_json.size()); - VG graph; - graph.extend(source); + vg::io::json2graph(graph_json, &graph); @@ -5405,11 +5392,8 @@ TEST_CASE("simplify_siblings() works on a smaller graph with a reversing self lo {"edge": [{"from": "1", "to": "3"}, {"from": "1", "to": "2"}, {"from": "2", "to": "2", "to_end": true}], "node": [{"id": "1", "sequence": "T"}, {"id": "2", "sequence": "A"}, {"id": "3", "sequence": "A"}], "path": [{"mapping": [{"edit": [{"from_length": 1, "to_length": 1}], "position": {"node_id": "1"}, "rank": "1"}, {"edit": [{"from_length": 1, "to_length": 1}], "position": {"node_id": "2"}, "rank": "2"}, {"edit": [{"from_length": 1, "to_length": 1}], "position": {"is_reverse": true, "node_id": "2"}, "rank": "3"}], "name": "x"}]} )"; - Graph source; - json2pb(source, graph_json.c_str(), graph_json.size()); - VG graph; - graph.extend(source); + vg::io::json2graph(graph_json, &graph); @@ -5425,11 +5409,8 @@ TEST_CASE("normalize() works on a graph with a reversing self loop", "[algorithm {"edge": [{"from": "1", "to": "3"}, {"from": "1", "to": "2"}, {"from": "2", "to": "2", "to_end": true}], "node": [{"id": "1", "sequence": "T"}, {"id": "2", "sequence": "A"}, {"id": "3", "sequence": "ACA"}], "path": [{"mapping": [{"edit": [{"from_length": 1, "to_length": 1}], "position": {"node_id": "1"}, "rank": "1"}, {"edit": [{"from_length": 1, "to_length": 1}], "position": {"node_id": "2"}, "rank": "2"}, {"edit": [{"from_length": 1, "to_length": 1}], "position": {"is_reverse": true, "node_id": "2"}, "rank": "3"}], "name": "x"}, {"mapping": [{"edit": [{"from_length": 1, "to_length": 1}], "position": {"node_id": "1"}, "rank": "1"}, {"edit": [{"from_length": 3, "to_length": 3}], "position": {"node_id": "3"}, "rank": "2"}], "name": "y"}]} )"; - Graph source; - json2pb(source, graph_json.c_str(), graph_json.size()); - VG graph; - graph.extend(source); + vg::io::json2graph(graph_json, &graph); diff --git a/src/unittest/vpkg.cpp b/src/unittest/vpkg.cpp index 51a849c446..977814ff9c 100644 --- a/src/unittest/vpkg.cpp +++ b/src/unittest/vpkg.cpp @@ -13,7 +13,7 @@ #include "xg.hpp" #include "../vg.hpp" #include "../snarl_seed_clusterer.hpp" -#include "vg/io/json2pb.h" +#include "../io/json2graph.hpp" #include #include #include @@ -50,12 +50,12 @@ TEST_CASE("We can read and write XG", "[vpkg][handlegraph][xg]") { )"; // Load the JSON - Graph proto_graph; - json2pb(proto_graph, graph_json.c_str(), graph_json.size()); - + bdsg::HashGraph hash_graph; + vg::io::json2graph(graph_json, &hash_graph); + // Build the xg index xg::XG xg_index; - xg_index.from_path_handle_graph(VG(proto_graph)); + xg_index.from_path_handle_graph(hash_graph); stringstream ss; @@ -148,13 +148,10 @@ TEST_CASE("We can read VG from a VPKG-wrapped stream as a VG", "[vpkg][handlegra {"id":2,"sequence":"ACA"}], "edge":[{"to":2,"from":1}]} )"; - - // Load the JSON - Graph proto_graph; - json2pb(proto_graph, graph_json.c_str(), graph_json.size()); - - // Build the VG - vg::VG vg_graph(proto_graph); + + // Load the JSON directly into VG + vg::VG vg_graph; + vg::io::json2graph(graph_json, &vg_graph); // Save it stringstream ss; @@ -179,13 +176,10 @@ TEST_CASE("We can read VG from a VPKG-wrapped stream as a HandleGraph which is a {"id":2,"sequence":"ACA"}], "edge":[{"to":2,"from":1}]} )"; - - // Load the JSON - Graph proto_graph; - json2pb(proto_graph, graph_json.c_str(), graph_json.size()); - - // Build the VG - vg::VG vg_graph(proto_graph); + + // Load the JSON directly into VG + vg::VG vg_graph; + vg::io::json2graph(graph_json, &vg_graph); // Save it stringstream ss; @@ -210,13 +204,10 @@ TEST_CASE("We can read VG from a VPKG-wrapped stream as a HandleGraph which is a TEST_CASE("We can read an empty VG as a HandleGraph", "[vpkg][handlegraph][vg][empty]") { string graph_json = "{}"; - - // Load the JSON - Graph proto_graph; - json2pb(proto_graph, graph_json.c_str(), graph_json.size()); - - // Build the VG - vg::VG vg_graph(proto_graph); + + // Load the JSON directly into VG + vg::VG vg_graph; + vg::io::json2graph(graph_json, &vg_graph); // Save it stringstream ss; @@ -240,13 +231,10 @@ TEST_CASE("We prefer to read a graph as the first provided type that matches", " {"id":2,"sequence":"ACA"}], "edge":[{"to":2,"from":1}]} )"; - - // Load the JSON - Graph proto_graph; - json2pb(proto_graph, graph_json.c_str(), graph_json.size()); - - // Build the VG - vg::VG vg_graph(proto_graph); + + // Load the JSON directly into VG + vg::VG vg_graph; + vg::io::json2graph(graph_json, &vg_graph); // Save it stringstream ss; diff --git a/src/unittest/xdrop_aligner.cpp b/src/unittest/xdrop_aligner.cpp index f745b8f66a..07577e4479 100644 --- a/src/unittest/xdrop_aligner.cpp +++ b/src/unittest/xdrop_aligner.cpp @@ -5,7 +5,7 @@ #include #include -#include "vg/io/json2pb.h" +#include "../io/json2graph.hpp" #include "../alignment.hpp" #include "../vg.hpp" #include @@ -764,12 +764,9 @@ TEST_CASE("QualAdjXdropAligner will not penalize a low quality mismatch", "[xdro TEST_CASE("XdropAligner doesn't crash on a case where it is hard to find a seed", "[xdrop][alignment][mapping]") { string graph_json = R"({"edge": [{"from": "92345167", "to": "92345168"}, {"from": "92345182", "to": "92345183"}, {"from": "92345165", "to": "92345166"}, {"from": "92345177", "to": "92345178"}, {"from": "92345171", "to": "92345172"}, {"from": "92345161", "to": "92345162"}, {"from": "92345183", "to": "92345184"}, {"from": "92345181", "to": "92345182"}, {"from": "92345178", "to": "92345179"}, {"from": "92345166", "to": "92345167"}, {"from": "92345179", "to": "92345180"}, {"from": "92345173", "to": "92345174"}, {"from": "92345184", "to": "92345185"}, {"from": "92345169", "to": "92345170"}, {"from": "92345185", "to": "92345186"}, {"from": "92345160", "to": "92345161"}, {"from": "92345174", "to": "92345175"}, {"from": "92345162", "to": "92345163"}, {"from": "92345175", "to": "92345176"}, {"from": "92345168", "to": "92345169"}, {"from": "92345163", "to": "92345164"}, {"from": "92345172", "to": "92345173"}, {"from": "92345180", "to": "92345181"}, {"from": "92345176", "to": "92345177"}, {"from": "92345170", "to": "92345171"}, {"from": "92345164", "to": "92345165"}], "node": [{"id": "92345167", "sequence": "TTTATATATATATATTTATATATATATATTTA"}, {"id": "92345182", "sequence": "TATATATATTTATATATATATTTATATATATA"}, {"id": "92345165", "sequence": "ATATATATATATTTATATATATTTATATATTA"}, {"id": "92345177", "sequence": "TTTATATATATATTTATATATATATATTATAT"}, {"id": "92345171", "sequence": "TTATATATATATTTATATATATATTTATATAT"}, {"id": "92345161", "sequence": "ATATATTTATATATTTTTATATATTATATATT"}, {"id": "92345183", "sequence": "TTTATATATATTTATATATATATTTATATATA"}, {"id": "92345181", "sequence": "ATATATTATATATATATTTATATATATATTTA"}, {"id": "92345178", "sequence": "ATATATTTATATATATATTTATATATATATTT"}, {"id": "92345166", "sequence": "TTTATATATATTTATATATATATTTATATATA"}, {"id": "92345179", "sequence": "ATATATATATTTATATATATATTTATATATAT"}, {"id": "92345173", "sequence": "ATATTTATATATATATATTTATATATATATTT"}, {"id": "92345184", "sequence": "TATTTATATATATATTTATATATATTTATATA"}, {"id": "92345169", "sequence": "TTTATATATATATTTATATATATATTTATATA"}, {"id": "92345185", "sequence": "TATATTTATATATATATATATATATTTATATA"}, {"id": "92345160", "sequence": "ATTTATATATATATTTATATATATATTTATAT"}, {"id": "92345174", "sequence": "ATATATATATTTATATATATATTATTTATATA"}, {"id": "92345162", "sequence": "TATATATATATTTATATATTATATATATATTT"}, {"id": "92345175", "sequence": "TATATTTATATATATATTATATATATATTTAT"}, {"id": "92345168", "sequence": "TATATATATTTATATATATATTTATATATATA"}, {"id": "92345163", "sequence": "ATATATTTATATATATATTTATATATATTTAT"}, {"id": "92345172", "sequence": "ATATATATATATTTATATATATATTTATATAT"}, {"id": "92345180", "sequence": "ATTTATATATATATTTATATATATATTTATAT"}, {"id": "92345176", "sequence": "ATATATATATTATATATATATTTATATATATA"}, {"id": "92345170", "sequence": "TATATTTATATATATATATTATATATATATAT"}, {"id": "92345164", "sequence": "ATATATATTTATATATATTTATATATATATTT"}, {"id": "92345186", "sequence": "TATATTTATATATATTTATATATATATTTATA"}]})"; - - Graph source; - json2pb(source, graph_json.c_str(), graph_json.size()); - - VG graph; - graph.extend(source); + + bdsg::HashGraph graph; + vg::io::json2graph(graph_json, &graph); Alignment aln; aln.set_sequence("CAGCACTTTGGGAGGCCAAGGTGGGTGGATCATCTGAGGTCAGGAGTTTGAGACCAGCCTGACCAACATGGTGAAATCCTGTCTCTACTGAAAATACTAAAATTAGCCAGGCGTGGCGGCCAGTGCCTGTAATCCCGGCTACTGGGGAGG"); diff --git a/src/unittest/xg.cpp b/src/unittest/xg.cpp index d74db5d0b0..dfa913b8eb 100644 --- a/src/unittest/xg.cpp +++ b/src/unittest/xg.cpp @@ -8,7 +8,9 @@ #include "vg.hpp" #include "xg.hpp" #include "graph.hpp" +#include "../io/json2graph.hpp" #include "algorithms/subgraph.hpp" +#include "bdsg/hash_graph.hpp" #include namespace vg { @@ -22,19 +24,18 @@ TEST_CASE("We can build an xg index on a nice graph", "[xg]") { {"id":2,"sequence":"ACA"}], "edge":[{"to":2,"from":1}]} )"; - + // Load the JSON - Graph proto_graph; - json2pb(proto_graph, graph_json.c_str(), graph_json.size()); - + bdsg::HashGraph source; + vg::io::json2graph(graph_json, &source); + // Build the xg index xg::XG xg_index; - xg_index.from_path_handle_graph(VG(proto_graph)); + xg_index.from_path_handle_graph(source); VG vg_graph; algorithms::extract_context(xg_index, vg_graph, xg_index.get_handle(1), 0, 100); Graph& graph = vg_graph.graph; - sort_by_id_dedup_and_clean(graph); REQUIRE(graph.node_size() == 2); REQUIRE(graph.edge_size() == 1); @@ -49,19 +50,18 @@ TEST_CASE("We can build an xg index on a nasty graph", "[xg]") { {"id":9999,"sequence":"AAA"}], "edge":[{"to":2,"from":1}]} )"; - + // Load the JSON - Graph proto_graph; - json2pb(proto_graph, graph_json.c_str(), graph_json.size()); - + bdsg::HashGraph source; + vg::io::json2graph(graph_json, &source); + // Build the xg index xg::XG xg_index; - xg_index.from_path_handle_graph(VG(proto_graph)); + xg_index.from_path_handle_graph(source); VG vg_graph; algorithms::extract_context(xg_index, vg_graph, xg_index.get_handle(1), 0, 100); Graph& graph = vg_graph.graph; - sort_by_id_dedup_and_clean(graph); REQUIRE(graph.node_size() == 2); REQUIRE(graph.edge_size() == 1); @@ -161,15 +161,14 @@ TEST_CASE("We can build an xg index on a very nasty graph", "[xg]") { {"position":{"node_id":1444},"rank":1059}, {"position":{"node_id":1445},"rank":1060}]}]} )"; - + // Load the JSON - Graph proto_graph; - json2pb(proto_graph, graph_json.c_str(), graph_json.size()); + VG source; + vg::io::json2graph(graph_json, &source); - sort_by_id_dedup_and_clean(proto_graph); // Build the xg index xg::XG xg_index; - xg_index.from_path_handle_graph(VG(proto_graph)); + xg_index.from_path_handle_graph(source); SECTION("Context extraction gets something") { VG graph; @@ -182,7 +181,7 @@ TEST_CASE("We can build an xg index on a very nasty graph", "[xg]") { SECTION("We can extract within a single node") { algorithms::extract_path_range(xg_index, xg_index.get_path_handle("17"), 5, 15, graph); - + // We should just get node 1416 REQUIRE(graph.graph.node_size() == 1); REQUIRE(graph.graph.node(0).id() == 1416); @@ -265,14 +264,14 @@ TEST_CASE("We can build and scan an XG index for a problematic graph", "[xg]") { ]} ]} )"; - + // Load the JSON - Graph proto_graph; - json2pb(proto_graph, graph_json.c_str(), graph_json.size()); + bdsg::HashGraph source; + vg::io::json2graph(graph_json, &source); // Build the xg index (without any sorting) xg::XG xg_index; - xg_index.from_path_handle_graph(VG(proto_graph)); + xg_index.from_path_handle_graph(source); REQUIRE(xg_index.get_node_count() == 5); @@ -300,18 +299,16 @@ TEST_CASE("We can build the xg index on a small graph with discontinuous node id )"; // Load the JSON - Graph proto_graph; - json2pb(proto_graph, graph_json.c_str(), graph_json.size()); + VG source; + vg::io::json2graph(graph_json, &source); - sort_by_id_dedup_and_clean(proto_graph); // Build the xg index xg::XG xg_index; - xg_index.from_path_handle_graph(VG(proto_graph)); + xg_index.from_path_handle_graph(source); VG vg_graph; algorithms::extract_context(xg_index, vg_graph, xg_index.get_handle(10), 0, 100); Graph& graph = vg_graph.graph; - sort_by_id_dedup_and_clean(graph); REQUIRE(graph.node_size() == 2); REQUIRE(graph.edge_size() == 1); @@ -326,14 +323,14 @@ TEST_CASE("Looping over XG handles in parallel works", "[xg]") { {"id":2,"sequence":"ACA"}], "edge":[{"to":2,"from":1}]} )"; - + // Load the JSON - Graph proto_graph; - json2pb(proto_graph, graph_json.c_str(), graph_json.size()); - + bdsg::HashGraph source; + vg::io::json2graph(graph_json, &source); + // Build the xg index xg::XG xg_index; - xg_index.from_path_handle_graph(VG(proto_graph)); + xg_index.from_path_handle_graph(source); size_t count = 0; @@ -341,7 +338,7 @@ TEST_CASE("Looping over XG handles in parallel works", "[xg]") { #pragma omp critical count++; }, true); - + REQUIRE(count == 2); } @@ -400,14 +397,14 @@ TEST_CASE("Vectorization of xg works correctly", "[xg]") { {"edit": [{"from_length": 11, "to_length": 11}], "position": {"node_id": "15"}, "rank": "10"} ], "name": "x"}]} )"; - + // Load the JSON - Graph proto_graph; - json2pb(proto_graph, graph_json.c_str(), graph_json.size()); + bdsg::HashGraph source; + vg::io::json2graph(graph_json, &source); // Build the xg index (without any sorting) xg::XG xg_index; - xg_index.from_path_handle_graph(VG(proto_graph)); + xg_index.from_path_handle_graph(source); REQUIRE(xg_index.get_node_count() == 15); From 904f4450bf64ba153f3373f9be382ac79a54651d Mon Sep 17 00:00:00 2001 From: Adam Novak Date: Tue, 10 Feb 2026 17:21:52 -0500 Subject: [PATCH 23/77] Remove duplicative JSON to graph function --- src/unittest/support/json.cpp | 24 ------------------------ src/unittest/support/json.hpp | 21 --------------------- 2 files changed, 45 deletions(-) delete mode 100644 src/unittest/support/json.cpp delete mode 100644 src/unittest/support/json.hpp diff --git a/src/unittest/support/json.cpp b/src/unittest/support/json.cpp deleted file mode 100644 index a3954ab746..0000000000 --- a/src/unittest/support/json.cpp +++ /dev/null @@ -1,24 +0,0 @@ -#include "json.hpp" - -#include "vg/io/json2pb.h" -#include "vg.hpp" - -namespace vg { -namespace unittest { - -std::unique_ptr json_to_graph(const std::string& json) { - // Load into a Protobuf object - Graph source; - json2pb(source, json.c_str(), json.size()); - - // Make a HandleGraph that knows how to load from Protobuf - auto to_return = std::make_unique(); - - // Load it from Protobuf - to_return->extend(source); - return to_return; -} - - -} -} diff --git a/src/unittest/support/json.hpp b/src/unittest/support/json.hpp deleted file mode 100644 index 63e51834a0..0000000000 --- a/src/unittest/support/json.hpp +++ /dev/null @@ -1,21 +0,0 @@ -#ifndef VG_UNITTEST_JSON_HPP_INCLUDED -#define VG_UNITTEST_JSON_HPP_INCLUDED -/** \file json.hpp - * Utilities for working with JSON data in test cases. - */ - -#include "handle.hpp" -#include - - -namespace vg { -namespace unittest { - -/// Create a handlegraph from vg Protobuf JSON. -std::unique_ptr json_to_graph(const std::string& json); - - -} -} - -#endif From 809a7665bb17e2f38b7dd790b708369de22f6824 Mon Sep 17 00:00:00 2001 From: Adam Novak Date: Tue, 10 Feb 2026 17:53:41 -0500 Subject: [PATCH 24/77] Set up tiny test that breaks oversized snarl logic --- src/unittest/snarl_distance_index.cpp | 58 +++++++++++++++++++++++++++ 1 file changed, 58 insertions(+) diff --git a/src/unittest/snarl_distance_index.cpp b/src/unittest/snarl_distance_index.cpp index 0c308a00b0..ec6600ce4f 100644 --- a/src/unittest/snarl_distance_index.cpp +++ b/src/unittest/snarl_distance_index.cpp @@ -7656,6 +7656,64 @@ namespace vg { // return true; // }); //} + + TEST_CASE( "Distance index can query a troublesome oversized snarl", + "[snarl_distance]" ) { + + std::string graph_json = R"({ + "node": [ + {"id": "19","sequence": "A"}, + {"id": "20","sequence": "A"}, + {"id": "21","sequence": "A"}, + {"id": "22","sequence": "A"}, + {"id": "23","sequence": "A"} + ], "edge": [ + {"from": "19","to": "20"}, + {"from": "19","to": "22"}, + {"from": "20","to": "21"}, + {"from": "20","to": "23"}, + {"from": "21","to": "22"}, + {"from": "22","to": "23"} + ] + })"; + + bdsg::HashGraph graph; + vg::io::json2graph(graph_json, &graph); + + IntegratedSnarlFinder snarl_finder(graph); + SnarlDistanceIndex distance_index; + fill_in_distance_index(&distance_index, &graph, &snarl_finder, 2); + + id_t node_id1 = 19; bool rev1 = false ; size_t offset1 = 0; + id_t node_id2 = 23; bool rev2 = false ; size_t offset2 = 0; + handle_t handle1 = graph.get_handle(node_id1, rev1); + handle_t handle2 = graph.get_handle(node_id2, rev2); + + //Find actual distance + size_t dijkstra_distance = std::numeric_limits::max(); + if (node_id1 == node_id2 && offset1 <= offset2 && rev1 == rev2) { + dijkstra_distance = offset2 - offset1; + REQUIRE(distance_index.minimum_distance(node_id1, rev1, offset1, node_id2, rev2, offset2, false, &graph) == dijkstra_distance); + } else if (node_id1 == node_id2) { + //TODO: The way the dijkstra algorithm is set up, it won't return to the start node + } else { + handlegraph::algorithms::dijkstra(&graph, handle1, [&](const handle_t& reached, size_t distance) { + if (reached == handle2) { + dijkstra_distance = distance; + dijkstra_distance += graph.get_length(graph.get_handle(node_id1)) - offset1; + dijkstra_distance += offset2; + return false; + } + return true; + } + , false); + + REQUIRE(distance_index.minimum_distance(node_id1, rev1, offset1, node_id2, rev2, offset2, false, &graph) == dijkstra_distance); + } + } + + + TEST_CASE( "random minimum distance paths", "[snarl_distance_random_paths]" ) { From f2d4f081fb17a019948611d254c6e9d53304a7f3 Mon Sep 17 00:00:00 2001 From: Adam Novak Date: Tue, 10 Feb 2026 17:54:07 -0500 Subject: [PATCH 25/77] Remove unused cases --- src/unittest/snarl_distance_index.cpp | 27 ++++++++++----------------- 1 file changed, 10 insertions(+), 17 deletions(-) diff --git a/src/unittest/snarl_distance_index.cpp b/src/unittest/snarl_distance_index.cpp index ec6600ce4f..0d7512ea7b 100644 --- a/src/unittest/snarl_distance_index.cpp +++ b/src/unittest/snarl_distance_index.cpp @@ -7691,25 +7691,18 @@ namespace vg { //Find actual distance size_t dijkstra_distance = std::numeric_limits::max(); - if (node_id1 == node_id2 && offset1 <= offset2 && rev1 == rev2) { - dijkstra_distance = offset2 - offset1; - REQUIRE(distance_index.minimum_distance(node_id1, rev1, offset1, node_id2, rev2, offset2, false, &graph) == dijkstra_distance); - } else if (node_id1 == node_id2) { - //TODO: The way the dijkstra algorithm is set up, it won't return to the start node - } else { - handlegraph::algorithms::dijkstra(&graph, handle1, [&](const handle_t& reached, size_t distance) { - if (reached == handle2) { - dijkstra_distance = distance; - dijkstra_distance += graph.get_length(graph.get_handle(node_id1)) - offset1; - dijkstra_distance += offset2; - return false; - } - return true; + handlegraph::algorithms::dijkstra(&graph, handle1, [&](const handle_t& reached, size_t distance) { + if (reached == handle2) { + dijkstra_distance = distance; + dijkstra_distance += graph.get_length(graph.get_handle(node_id1)) - offset1; + dijkstra_distance += offset2; + return false; } - , false); - - REQUIRE(distance_index.minimum_distance(node_id1, rev1, offset1, node_id2, rev2, offset2, false, &graph) == dijkstra_distance); + return true; } + , false); + + REQUIRE(distance_index.minimum_distance(node_id1, rev1, offset1, node_id2, rev2, offset2, false, &graph) == dijkstra_distance); } From caaa51276c622363d528cc3ee1353b3484ae9b1c Mon Sep 17 00:00:00 2001 From: Adam Novak Date: Tue, 10 Feb 2026 19:54:08 -0500 Subject: [PATCH 26/77] Fill in the dustances through oversized snarls to pass more distance indexing test cases --- deps/libbdsg | 2 +- src/snarl_distance_index.cpp | 49 +++++++++++++++++++++--------- src/subcommand/haplotypes_main.cpp | 2 +- 3 files changed, 36 insertions(+), 17 deletions(-) diff --git a/deps/libbdsg b/deps/libbdsg index 9dc03260a5..d429581e17 160000 --- a/deps/libbdsg +++ b/deps/libbdsg @@ -1 +1 @@ -Subproject commit 9dc03260a52c263708a92a41096989cc5cfdbbaa +Subproject commit d429581e179d11857c7dd5bfab62409e5e7f6fdf diff --git a/src/snarl_distance_index.cpp b/src/snarl_distance_index.cpp index 2eafabb500..16193b931e 100644 --- a/src/snarl_distance_index.cpp +++ b/src/snarl_distance_index.cpp @@ -788,26 +788,31 @@ SnarlDistanceIndex::TemporaryDistanceIndex make_temporary_distance_index( return temp_index; } +/** + * Populate a row of the distance matrix. + * Also responsible for filling in min_length, distance_start_start, and distance_start_end on the TemporarySnarlRecord when a distance matrix is used. + */ static void populate_distance_matrix_row(SnarlDistanceIndex::TemporaryDistanceIndex& temp_index, SnarlDistanceIndex::temp_record_ref_t& snarl_index, SnarlDistanceIndex::TemporaryDistanceIndex::TemporarySnarlRecord& temp_snarl_record, const SnarlDistanceIndex::temp_record_ref_t& start_index, const HandleGraph* graph, size_t start_rank, bool is_internal_node, size_t size_limit); -/* -Fills in required distance matrix rows for each child -- Normal snarl: all rows -- Oversized snarl: boundaries and tips -- size_limit == 0: no distances in index, so no rows -- Top-level chain distances only: ??? -*/ +/** + * Fills in required distance matrix rows for each child + * - Normal snarl: all rows + * - Oversized snarl: boundaries and tips + * - size_limit == 0: no distances in index, so no rows + * - Top-level chain distances only: ??? + */ static void populate_distance_matrix_if_needed(SnarlDistanceIndex::TemporaryDistanceIndex& temp_index, SnarlDistanceIndex::temp_record_ref_t& snarl_index, SnarlDistanceIndex::TemporaryDistanceIndex::TemporarySnarlRecord& temp_snarl_record, vector& all_children, const HandleGraph* graph, size_t size_limit, bool only_top_level_chain_distances); -/* -Does three things: -- Builds temp graph that hub labels will be built on -- Builds the hub labels -- Stores labels in temp_snarl_record -*/ +/** + * Does three things: + * - Builds temp graph that hub labels will be built on + * - Builds the hub labels + * - Stores labels in temp_snarl_record + */ static void populate_hub_labeling(SnarlDistanceIndex::TemporaryDistanceIndex& temp_index, SnarlDistanceIndex::temp_record_ref_t& snarl_index, SnarlDistanceIndex::TemporaryDistanceIndex::TemporarySnarlRecord& temp_snarl_record, vector& all_children, const HandleGraph* graph); -/*Fill in the snarl index. +/** + * Fill in the snarl index. * The index will already know its boundaries and everything knows their relationships in the * snarl tree. This needs to fill in the distances and the ranks of children in the snarl * The rank of a child is arbitrary, except that the start node will always be 0 and the end node @@ -1058,11 +1063,25 @@ void populate_snarl_index( temp_index.use_oversized_snarls = true; temp_snarl_record.is_simple = false; populate_hub_labeling(temp_index, snarl_index, temp_snarl_record, all_children, graph); + + // We need to query the hub labeling to fill in min_length, + // distance_start_start, and distance_start_end with the connectivity + // distances through the snarl, not including boundary nodes. + // + // Luckily we know the start is always child rank 0 forward, and the end + // is always child rank 1 forward. + // + // To exclude the boundary lengths we go from source port to non-source + // port. + temp_snarl_record.min_length = promote_distance(hhl_query(temp_snarl_record.hub_labels.begin(), bgid(0, false, true), bgid(1, false, false))); + temp_snarl_record.distance_start_start = promote_distance(hhl_query(temp_snarl_record.hub_labels.begin(), bgid(0, false, true), bgid(0, true, false))); + temp_snarl_record.distance_end_end = promote_distance(hhl_query(temp_snarl_record.hub_labels.begin(), bgid(1, true, true), bgid(1, false, false))); + // TODO: Should this be here or should it be part of populate_hub_labeling()? Or its own function? } else { if (size_limit == 0 || only_top_level_chain_distances) { temp_snarl_record.include_distances = false; } - //also sets is_simple to false if snarl isn't simple + //Also fills in min_lenght, distance_start_start, and distance_start_end, and sets is_simple to false if snarl isn't simple populate_distance_matrix_if_needed(temp_index, snarl_index, temp_snarl_record, all_children, graph, size_limit, only_top_level_chain_distances); } diff --git a/src/subcommand/haplotypes_main.cpp b/src/subcommand/haplotypes_main.cpp index d53dac99ae..0919715f6c 100644 --- a/src/subcommand/haplotypes_main.cpp +++ b/src/subcommand/haplotypes_main.cpp @@ -867,7 +867,7 @@ std::string pair_to_string(std::pair value) { } void validate_error_chain(const Logger& logger, size_t chain_id, const std::string& message) { - logger.error() << "[chain " << chain_id + "] " << message << std::endl; + logger.error() << "[chain " << chain_id << "] " << message << std::endl; } void validate_error_subchain(const Logger& logger, size_t chain_id, size_t subchain_id, const std::string& message) { From f15a5f9f7db037a44ab5aec2f52346f851e7182e Mon Sep 17 00:00:00 2001 From: Adam Novak Date: Wed, 11 Feb 2026 12:59:51 -0500 Subject: [PATCH 27/77] Add exhaustive test for small snarls --- src/unittest/snarl_distance_index.cpp | 205 +++++++++++++++++++++++++- 1 file changed, 203 insertions(+), 2 deletions(-) diff --git a/src/unittest/snarl_distance_index.cpp b/src/unittest/snarl_distance_index.cpp index 0d7512ea7b..cf7d13637b 100644 --- a/src/unittest/snarl_distance_index.cpp +++ b/src/unittest/snarl_distance_index.cpp @@ -21,6 +21,8 @@ #include #include #include "xg.hpp" +#include +#include #define debug @@ -7423,7 +7425,7 @@ namespace vg { */ TEST_CASE( "Distance index can traverse all the snarls in random graphs", - "[snarl_distance_random]" ) { + "[snarl_distance][snarl_distance_random]" ) { // Each actual graph takes a fairly long time to do so we randomize sizes... @@ -7705,10 +7707,209 @@ namespace vg { REQUIRE(distance_index.minimum_distance(node_id1, rev1, offset1, node_id2, rev2, offset2, false, &graph) == dijkstra_distance); } + + TEST_CASE( "Distance index can query all possible 3-node-with-legs snarls", + "[snarl_distance]" ) { + + // We're going to generate all possible snarls you can get by + // starting with the boundary nodes, taking up to 3 nodes and + // connecting them, one nodeside at a time, onto the existing + // nodes. + // + // Combinatorics says this is a manageable number; each nodeside + // picks from one of the previous nodesides and attaches to it. + + /// Call the callback with each possible combination of choices of + /// previous items. + /// + /// start_size is the number of items present before we start + /// making choices; the first entry can choose from start_size + /// items. + /// + /// end_size is the total number of items to think about, including + /// those in start_size. + /// + /// Calls the callback with all possible vectors of length + /// (end_size - start_size) matching these constraints. + auto for_all_choice_combinations = [](size_t start_size, size_t end_size, const std::function&)>& callback) { + + std::vector choices(end_size - start_size, 0); + while (true) { + std::cerr << "Consider combination:"; + for (auto& item : choices) { + std::cerr << " " << item; + } + std::cerr << std::endl; + callback(choices); + choices.back()++; + for (size_t i = end_size - 1; i >= start_size; i--) { + if (choices.at(i - start_size) >= i) { + // We've reached the point where we want to pick from a + // choice not available at this point. + // At i=2 we can choose between 0 and 1, so we carry at i. + if (i == start_size) { + // We've counted all possibilities + return; + } else { + // Carry and reset to 0. + choices.at(i - start_size - 1)++; + choices.at(i - start_size) = 0; + } + } else { + // No more carrying to do + break; + } + } + } + }; + + // How big should a snarl be allowed to be before being oversized? + size_t size_limit = 2; + // How many content nodes should be inside the snarl? + const size_t MAX_NODES = 3; + // How many node sides do we need to worry about, including the boundary sentinels? + size_t max_node_sides = MAX_NODES * 2 + 2; + for_all_choice_combinations(2, max_node_sides, [&](const std::vector& choices) { + // Build the choices into a graph. + + bdsg::HashGraph graph; + // Make the bounding nodes heavy so they are likely to root the snarl + handle_t start_node = graph.create_handle("AAAAA"); + handle_t end_node = graph.create_handle("AAAAA"); + + std::vector connect_to; + connect_to.reserve(max_node_sides); + // Choice 0 is start node, arriving reading out + connect_to.push_back(graph.flip(start_node)); + // Choice 1 is end node reading out + connect_to.push_back(end_node); + + for (size_t i = 0; i < choices.size(); i += 2) { + // Make a node + handle_t new_node = graph.create_handle("A"); + // Make sure to remember it so it can choose itself + connect_to.push_back(new_node); + connect_to.push_back(graph.flip(new_node)); + // Connect its left and right to each pair of choices. + graph.create_edge(graph.flip(new_node), connect_to.at(choices.at(i))); + graph.create_edge(new_node, connect_to.at(choices.at(i + 1))); + } + + // TODO: It might be more efficient to un-build the things that + // change between graphs instead of rebuilding from scratch for + // every case. + + // Skip graphs where the choices mean the graph isn't actually + // connected, because then it can't be recognized as a snarl + // probably. + std::vector> components = handlegraph::algorithms::weakly_connected_components(&graph); + if (components.size() > 1) { + return; + } + + // Now index the graph for query + IntegratedSnarlFinder finder(graph); + SnarlDistanceIndex distance_index; + fill_in_distance_index(&distance_index, &graph, &finder, size_limit); + + // Compute the truth all-to-all distances, between outgoing + // side of first handle and incoming side of second. + // Both handles are oriented along the connecting path. + // TODO: We compute/store both triangles of the matrix; can we avoid one somehow? + std::unordered_map> dijkstra_distances; + graph.for_each_handle([&](const handle_t& base) { + for (const handle_t& here : {base, graph.flip(base)}) { + if (here == graph.flip(start_node) || here == end_node) { + // Skip traversals looking out of the snarl + return; + } + dijkstra_distances.emplace(here, handlegraph::algorithms::find_shortest_paths(&graph, here)); + } + }); + + // The Dijkstra traversal always sees a handle to itself at + // distance 0. We need to get the real back-to-self distance, + // if any, and fill that in. + graph.for_each_handle([&](const handle_t& base) { + for (const handle_t& here : {base, graph.flip(base)}) { + if (here == graph.flip(start_node) || here == end_node) { + // Skip traversals looking out of the snarl + return; + } + + // The place we need to arrive at is ourselves, since + // both start and end are oriented along the connecting + // path here. + + size_t loop_distance = std::numeric_limits::max(); + // See if we can get back here from any of the places we can get + graph.follow_edges(here, false, [&](const handle_t next) { + if (next == here) { + // We found a real self loop + loop_distance = 0; + return false; + } + auto found_index = dijkstra_distances.find(next); + if (found_index == dijkstra_distances.end()) { + // This destination can't get anywhere. + // This should be impossible since the Dijkstra always will point a node at itself. + return true; + } + auto found_distance = found_index->second.find(here); + if (found_distance == found_index->second.end()) { + // This destination can't get back to us + return true; + } + // If we find a way back, min in its distance. + loop_distance = std::min(loop_distance, graph.get_length(next) + found_distance->second); + return true; + }); + + std::cerr << "Real self loop distance for " << graph.get_id(here) << (graph.get_is_reverse(here) ? "rev" : "fd") << " -> " << graph.get_id(here) << (graph.get_is_reverse(here) ? "rev" : "fd") << " is " << loop_distance << std::endl; + + if (loop_distance == std::numeric_limits::max()) { + // There's really no way back from this node to itself in the same orientation. Delete the entry the Dijkstra search adds. + dijkstra_distances.at(here).erase(here); + } else { + // There is a way back; store the value. + dijkstra_distances.at(here)[here] = loop_distance; + } + }; + }); + + for (auto& [start_handle, distances] : dijkstra_distances) { + for (auto& [end_handle, dijkstra_distance] : distances) { + cerr << "Dijkstra sees: " << graph.get_id(start_handle) << (graph.get_is_reverse(start_handle) ? "rev" : "fd") << graph.get_length(start_handle) << " -> " << graph.get_id(end_handle) << (graph.get_is_reverse(end_handle) ? "rev" : "fd") << 0 << " = " << dijkstra_distance << endl; + } + } + + // Now query all of the distances against the index + for (auto& [start_handle, distances] : dijkstra_distances) { + for (auto& [end_handle, dijkstra_distance] : distances) { + // Ask for distance between outgoing side of first handle and incoming side of second. + + cerr << "Measure: " << graph.get_id(start_handle) << (graph.get_is_reverse(start_handle) ? "rev" : "fd") << graph.get_length(start_handle) << " -> " << graph.get_id(end_handle) << (graph.get_is_reverse(end_handle) ? "rev" : "fd") << 0 << endl; + + size_t snarl_distance = distance_index.minimum_distance(graph.get_id(start_handle), graph.get_is_reverse(start_handle), graph.get_length(start_handle), graph.get_id(end_handle), graph.get_is_reverse(end_handle), 0, false, &graph); + + if (snarl_distance != dijkstra_distance) { + cerr << "Failed exhaustive test" << endl; + cerr << "Snarl size limit: " << size_limit << endl; + cerr << graph.get_id(start_handle) << (graph.get_is_reverse(start_handle) ? "rev" : "fd") << graph.get_length(start_handle) << " -> " << graph.get_id(end_handle) << (graph.get_is_reverse(end_handle) ? "rev" : "fd") << 0 << endl; + cerr << "guessed: " << snarl_distance << " actual: " << dijkstra_distance << endl; + cerr << "serializing graph to test_graph.vg" << endl; + vg::io::VPKG::save(graph, "test_graph.vg"); + } + REQUIRE(snarl_distance == dijkstra_distance); + } + } + }); + + } TEST_CASE( "random minimum distance paths", - "[snarl_distance_random_paths]" ) { + "[snarl_distance][snarl_distance_random_paths]" ) { // Each actual graph takes a fairly long time to do so we randomize sizes... From a0c71e6f1b717a4110a0e0d3b6d3ec43b8eb011f Mon Sep 17 00:00:00 2001 From: Adam Novak Date: Wed, 11 Feb 2026 14:26:49 -0500 Subject: [PATCH 28/77] Add a test for one of the failing possible small oversized snarls specifically --- deps/libbdsg | 2 +- src/snarl_distance_index.cpp | 6 ++-- src/unittest/snarl_distance_index.cpp | 41 +++++++++++++++++++++++++++ 3 files changed, 45 insertions(+), 4 deletions(-) diff --git a/deps/libbdsg b/deps/libbdsg index d429581e17..43a6bc7a0f 160000 --- a/deps/libbdsg +++ b/deps/libbdsg @@ -1 +1 @@ -Subproject commit d429581e179d11857c7dd5bfab62409e5e7f6fdf +Subproject commit 43a6bc7a0f45d588f4c0dc75b8231b1797561f5f diff --git a/src/snarl_distance_index.cpp b/src/snarl_distance_index.cpp index 16193b931e..943c593d7d 100644 --- a/src/snarl_distance_index.cpp +++ b/src/snarl_distance_index.cpp @@ -1,8 +1,8 @@ -//#define debug_distance_indexing +#define debug_distance_indexing //#define debug_snarl_traversal -//#define debug_distances +#define debug_distances //#define debug_subgraph -//#define debug_hub_label_build +#define debug_hub_label_build //#define debug_hub_label_storage #include "snarl_distance_index.hpp" diff --git a/src/unittest/snarl_distance_index.cpp b/src/unittest/snarl_distance_index.cpp index cf7d13637b..68ad6cfe7f 100644 --- a/src/unittest/snarl_distance_index.cpp +++ b/src/unittest/snarl_distance_index.cpp @@ -7707,6 +7707,47 @@ namespace vg { REQUIRE(distance_index.minimum_distance(node_id1, rev1, offset1, node_id2, rev2, offset2, false, &graph) == dijkstra_distance); } + TEST_CASE( "Distance index can query out of a SNP with a reversing allele as an oversided snarl", + "[snarl_distance]" ) { + + // This is a snarl from 1 to 2, where 4 nand 5 are a SNP, and 3 + // lets you double back to the start + std::string graph_json = R"({ + "node": [ + {"id": "1","sequence": "AAAAA"}, + {"id": "2","sequence": "AAAAA"}, + {"id": "3","sequence": "A"}, + {"id": "4","sequence": "A"}, + {"id": "5","sequence": "A"} + ], "edge": [ + {"from": "1","to": "3"}, + {"from": "1","to": "4"}, + {"from": "1","to": "5"}, + {"from": "3","to": "1", "to_end": true}, + {"from": "4","to": "2"}, + {"from": "5","to": "2"} + ] + })"; + + bdsg::HashGraph graph; + vg::io::json2graph(graph_json, &graph); + + IntegratedSnarlFinder snarl_finder(graph); + SnarlDistanceIndex distance_index; + fill_in_distance_index(&distance_index, &graph, &snarl_finder, 2); + + // We want to be able to get out of the snarl from node 4, which we definitely can. + id_t node_id1 = 4; bool rev1 = false ; size_t offset1 = 1; + id_t node_id2 = 2; bool rev2 = false ; size_t offset2 = 0; + handle_t handle1 = graph.get_handle(node_id1, rev1); + handle_t handle2 = graph.get_handle(node_id2, rev2); + + //Find actual distance + size_t true_distance = 0; + + REQUIRE(distance_index.minimum_distance(node_id1, rev1, offset1, node_id2, rev2, offset2, false, &graph) == true_distance); + } + TEST_CASE( "Distance index can query all possible 3-node-with-legs snarls", "[snarl_distance]" ) { From 8c048e2324a0919cdaa47306236bf56e0ba43dae Mon Sep 17 00:00:00 2001 From: Adam Novak Date: Wed, 11 Feb 2026 17:37:16 -0500 Subject: [PATCH 29/77] Pin down one small graph --- deps/libbdsg | 2 +- src/unittest/snarl_distance_index.cpp | 7 +++++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/deps/libbdsg b/deps/libbdsg index 43a6bc7a0f..baa9b49109 160000 --- a/deps/libbdsg +++ b/deps/libbdsg @@ -1 +1 @@ -Subproject commit 43a6bc7a0f45d588f4c0dc75b8231b1797561f5f +Subproject commit baa9b4910952cd3208793df4f39594c3b4ac0cd1 diff --git a/src/unittest/snarl_distance_index.cpp b/src/unittest/snarl_distance_index.cpp index 68ad6cfe7f..878d793f5e 100644 --- a/src/unittest/snarl_distance_index.cpp +++ b/src/unittest/snarl_distance_index.cpp @@ -7746,6 +7746,13 @@ namespace vg { size_t true_distance = 0; REQUIRE(distance_index.minimum_distance(node_id1, rev1, offset1, node_id2, rev2, offset2, false, &graph) == true_distance); + + // And out of the snarl to the left from 3 reverse to 1 reverse should also be 0 + node_id1 = 3; rev1 = true; offset1 = 1; + node_id2 = 1; rev2 = true; offset2 = 0; + true_distance = 0; + REQUIRE(distance_index.minimum_distance(node_id1, rev1, offset1, node_id2, rev2, offset2, false, &graph) == true_distance); + } From e036967158ffde24129fa90f570e6a9b78ca1427 Mon Sep 17 00:00:00 2001 From: Adam Novak Date: Fri, 13 Feb 2026 12:56:25 -0500 Subject: [PATCH 30/77] Add more test cases from sequential and random graphs and make them pass --- deps/libbdsg | 2 +- src/unittest/snarl_distance_index.cpp | 68 ++++++++++++++++++++++++++- 2 files changed, 68 insertions(+), 2 deletions(-) diff --git a/deps/libbdsg b/deps/libbdsg index baa9b49109..ee08df26a4 160000 --- a/deps/libbdsg +++ b/deps/libbdsg @@ -1 +1 @@ -Subproject commit baa9b4910952cd3208793df4f39594c3b4ac0cd1 +Subproject commit ee08df26a4828936db0fbfc53ba75243a803a36f diff --git a/src/unittest/snarl_distance_index.cpp b/src/unittest/snarl_distance_index.cpp index 878d793f5e..3c65198c35 100644 --- a/src/unittest/snarl_distance_index.cpp +++ b/src/unittest/snarl_distance_index.cpp @@ -7755,6 +7755,64 @@ namespace vg { } + TEST_CASE( "Distance index can query within a fiddly snarl", + "[snarl_distance]" ) { + + std::string graph_json = R"({"edge": [{"from": "1", "to": "3"}, {"from": "1", "to": "3", "to_end": true}, {"from": "1", "to": "4"}, {"from": "1", "to": "5"}, {"from": "4", "to": "5", "to_end": true}, {"from": "2", "from_start": true, "to": "4", "to_end": true}], "node": [{"id": "5", "sequence": "A"}, {"id": "1", "sequence": "AAAAA"}, {"id": "4", "sequence": "A"}, {"id": "2", "sequence": "AAAAA"}, {"id": "3", "sequence": "A"}]})"; + + bdsg::HashGraph graph; + vg::io::json2graph(graph_json, &graph); + + IntegratedSnarlFinder snarl_finder(graph); + SnarlDistanceIndex distance_index; + fill_in_distance_index(&distance_index, &graph, &snarl_finder, 2); + + id_t node_id1 = 4; bool rev1 = false ; size_t offset1 = 1; + id_t node_id2 = 5; bool rev2 = true ; size_t offset2 = 0; + handle_t handle1 = graph.get_handle(node_id1, rev1); + handle_t handle2 = graph.get_handle(node_id2, rev2); + + //Find actual distance + size_t true_distance = 0; + + REQUIRE(distance_index.minimum_distance(node_id1, rev1, offset1, node_id2, rev2, offset2, false, &graph) == true_distance); + } + + TEST_CASE( "Distance index can query into a child snarl in reverse", + "[snarl_distance]" ) { + + std::string graph_json = R"({"node":[{"id":"79","sequence":"A"},{"id":"16","sequence":"A"},{"id":"60","sequence":"A"},{"id":"37","sequence":"A"},{"id":"40","sequence":"A"},{"id":"53","sequence":"A"},{"id":"59","sequence":"A"},{"id":"63","sequence":"A"},{"id":"18","sequence":"A"},{"id":"38","sequence":"A"},{"id":"62","sequence":"A"}],"edge":[{"from":"16","to":"53"},{"from":"16","from_start":true,"to":"79","to_end":true},{"from":"60","to":"62"},{"from":"60","from_start":true,"to":"79","to_end":true},{"from":"37","from_start":true,"to":"63","to_end":true},{"from":"37","from_start":true,"to":"40"},{"from":"53","to":"60"},{"from":"59","to":"63"},{"from":"59","from_start":true,"to":"60","to_end":true},{"from":"18","to":"53"},{"from":"18","to":"38"},{"from":"18","from_start":true,"to":"79","to_end":true},{"from":"18","from_start":true,"to":"37","to_end":true},{"from":"38","to":"63","to_end":true},{"from":"38","to":"40"},{"from":"62","to":"63"}]})"; + + bdsg::HashGraph graph; + vg::io::json2graph(graph_json, &graph); + + IntegratedSnarlFinder snarl_finder(graph); + SnarlDistanceIndex distance_index; + fill_in_distance_index(&distance_index, &graph, &snarl_finder, 2); + + id_t node_id1 = 16; bool rev1 = false ; size_t offset1 = 1; + id_t node_id2 = 62; bool rev2 = true ; size_t offset2 = 0; + handle_t handle1 = graph.get_handle(node_id1, rev1); + handle_t handle2 = graph.get_handle(node_id2, rev2); + + //Find actual distance + size_t dijkstra_distance = std::numeric_limits::max(); + handlegraph::algorithms::dijkstra(&graph, handle1, [&](const handle_t& reached, size_t distance) { + if (reached == handle2) { + dijkstra_distance = distance; + dijkstra_distance += graph.get_length(graph.get_handle(node_id1)) - offset1; + dijkstra_distance += offset2; + return false; + } + return true; + } + , false); + + size_t index_distance = distance_index.minimum_distance(node_id1, rev1, offset1, node_id2, rev2, offset2, false, &graph); + + REQUIRE(index_distance == dijkstra_distance); + } + TEST_CASE( "Distance index can query all possible 3-node-with-legs snarls", "[snarl_distance]" ) { @@ -7783,11 +7841,13 @@ namespace vg { std::vector choices(end_size - start_size, 0); while (true) { +#ifdef debug std::cerr << "Consider combination:"; for (auto& item : choices) { std::cerr << " " << item; } std::cerr << std::endl; +#endif callback(choices); choices.back()++; for (size_t i = end_size - 1; i >= start_size; i--) { @@ -7913,7 +7973,9 @@ namespace vg { return true; }); +#ifdef debug std::cerr << "Real self loop distance for " << graph.get_id(here) << (graph.get_is_reverse(here) ? "rev" : "fd") << " -> " << graph.get_id(here) << (graph.get_is_reverse(here) ? "rev" : "fd") << " is " << loop_distance << std::endl; +#endif if (loop_distance == std::numeric_limits::max()) { // There's really no way back from this node to itself in the same orientation. Delete the entry the Dijkstra search adds. @@ -7925,18 +7987,22 @@ namespace vg { }; }); +#ifdef debug for (auto& [start_handle, distances] : dijkstra_distances) { for (auto& [end_handle, dijkstra_distance] : distances) { cerr << "Dijkstra sees: " << graph.get_id(start_handle) << (graph.get_is_reverse(start_handle) ? "rev" : "fd") << graph.get_length(start_handle) << " -> " << graph.get_id(end_handle) << (graph.get_is_reverse(end_handle) ? "rev" : "fd") << 0 << " = " << dijkstra_distance << endl; } } +#endif // Now query all of the distances against the index for (auto& [start_handle, distances] : dijkstra_distances) { for (auto& [end_handle, dijkstra_distance] : distances) { // Ask for distance between outgoing side of first handle and incoming side of second. - + +#ifdef debug cerr << "Measure: " << graph.get_id(start_handle) << (graph.get_is_reverse(start_handle) ? "rev" : "fd") << graph.get_length(start_handle) << " -> " << graph.get_id(end_handle) << (graph.get_is_reverse(end_handle) ? "rev" : "fd") << 0 << endl; +#endif size_t snarl_distance = distance_index.minimum_distance(graph.get_id(start_handle), graph.get_is_reverse(start_handle), graph.get_length(start_handle), graph.get_id(end_handle), graph.get_is_reverse(end_handle), 0, false, &graph); From 8860e3aa4cd6e84f17d5aa4a07b84843d41c3ca2 Mon Sep 17 00:00:00 2001 From: Adam Novak Date: Fri, 13 Feb 2026 13:09:36 -0500 Subject: [PATCH 31/77] Turn off debugging after passing random graph test --- deps/libbdsg | 2 +- src/snarl_distance_index.cpp | 6 +++--- src/unittest/snarl_distance_index.cpp | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/deps/libbdsg b/deps/libbdsg index ee08df26a4..f522ff9a20 160000 --- a/deps/libbdsg +++ b/deps/libbdsg @@ -1 +1 @@ -Subproject commit ee08df26a4828936db0fbfc53ba75243a803a36f +Subproject commit f522ff9a20654627b9a803ba4b0fd33cc4f54dbc diff --git a/src/snarl_distance_index.cpp b/src/snarl_distance_index.cpp index 943c593d7d..16193b931e 100644 --- a/src/snarl_distance_index.cpp +++ b/src/snarl_distance_index.cpp @@ -1,8 +1,8 @@ -#define debug_distance_indexing +//#define debug_distance_indexing //#define debug_snarl_traversal -#define debug_distances +//#define debug_distances //#define debug_subgraph -#define debug_hub_label_build +//#define debug_hub_label_build //#define debug_hub_label_storage #include "snarl_distance_index.hpp" diff --git a/src/unittest/snarl_distance_index.cpp b/src/unittest/snarl_distance_index.cpp index 3c65198c35..784a8f714a 100644 --- a/src/unittest/snarl_distance_index.cpp +++ b/src/unittest/snarl_distance_index.cpp @@ -24,7 +24,7 @@ #include #include -#define debug +//#define debug namespace vg { namespace unittest { From eeaa175dc11dc1ee1b06b4eeac1461e166b04be0 Mon Sep 17 00:00:00 2001 From: Adam Novak Date: Fri, 13 Feb 2026 15:57:17 -0500 Subject: [PATCH 32/77] Add initial synthetic code for SnarlDecompositionFuzzer and randomly_flipped_nodes --- src/unittest/randomly_flipped_nodes.cpp | 192 ++++++++ src/unittest/snarl_decomposition_fuzzer.cpp | 448 ++++++++++++++++++ .../support/randomly_flipped_nodes.hpp | 83 ++++ .../support/snarl_decomposition_fuzzer.cpp | 274 +++++++++++ .../support/snarl_decomposition_fuzzer.hpp | 139 ++++++ 5 files changed, 1136 insertions(+) create mode 100644 src/unittest/randomly_flipped_nodes.cpp create mode 100644 src/unittest/snarl_decomposition_fuzzer.cpp create mode 100644 src/unittest/support/randomly_flipped_nodes.hpp create mode 100644 src/unittest/support/snarl_decomposition_fuzzer.cpp create mode 100644 src/unittest/support/snarl_decomposition_fuzzer.hpp diff --git a/src/unittest/randomly_flipped_nodes.cpp b/src/unittest/randomly_flipped_nodes.cpp new file mode 100644 index 0000000000..0c1d158d24 --- /dev/null +++ b/src/unittest/randomly_flipped_nodes.cpp @@ -0,0 +1,192 @@ +#include "catch.hpp" +#include "../handle.hpp" +#include "../utility.hpp" +#include + +#include "support/randomly_flipped_nodes.hpp" +#include "support/randomness.hpp" +#include "support/random_graph.hpp" + +#include +#include + +namespace vg { +namespace unittest { + +using namespace std; + +/// Get the canonicalized set of edge sequence pairs from a graph. +/// Each edge is represented as a pair of sequences (left_seq, right_seq) read +/// in the orientation of the edge. To canonicalize, we compare each pair +/// against its reverse complement (RC(right_seq), RC(left_seq)) and keep the +/// lexicographically smaller one. +static set> canonical_edge_pairs(const HandleGraph& graph) { + set> result; + graph.for_each_edge([&](const edge_t& edge) { + string left_seq = graph.get_sequence(edge.first); + string right_seq = graph.get_sequence(edge.second); + + // The reverse complement pair: RC(right) on the left, RC(left) on the right + string rc_right = reverse_complement(right_seq); + string rc_left = reverse_complement(left_seq); + + pair forward_pair = {left_seq, right_seq}; + pair rc_pair = {rc_right, rc_left}; + + // Use the lexicographically smaller one as canonical + if (rc_pair < forward_pair) { + result.insert(rc_pair); + } else { + result.insert(forward_pair); + } + return true; + }); + return result; +} + +TEST_CASE("randomly_flipped_nodes preserves graph structure on a simple linear graph", "[randomly_flipped_nodes]") { + bdsg::HashGraph graph; + handle_t h1 = graph.create_handle("AAA", 1); + handle_t h2 = graph.create_handle("CGC", 2); + handle_t h3 = graph.create_handle("TTT", 3); + graph.create_edge(h1, h2); + graph.create_edge(h2, h3); + + auto original_edges = canonical_edge_pairs(graph); + + SECTION("flipping no nodes preserves edges exactly") { + default_random_engine gen(test_seed_source()); + auto flipped = randomly_flipped_nodes(graph, 0.0, gen); + + REQUIRE(flipped.get_node_count() == 3); + REQUIRE(flipped.get_edge_count() == 2); + + auto flipped_edges = canonical_edge_pairs(flipped); + REQUIRE(original_edges == flipped_edges); + } + + SECTION("flipping all nodes preserves canonical edge pairs") { + default_random_engine gen(test_seed_source()); + auto flipped = randomly_flipped_nodes(graph, 1.0, gen); + + REQUIRE(flipped.get_node_count() == 3); + REQUIRE(flipped.get_edge_count() == 2); + + auto flipped_edges = canonical_edge_pairs(flipped); + REQUIRE(original_edges == flipped_edges); + } + + SECTION("flipping 50% of nodes preserves canonical edge pairs") { + default_random_engine gen(test_seed_source()); + auto flipped = randomly_flipped_nodes(graph, 0.5, gen); + + REQUIRE(flipped.get_node_count() == 3); + REQUIRE(flipped.get_edge_count() == 2); + + auto flipped_edges = canonical_edge_pairs(flipped); + REQUIRE(original_edges == flipped_edges); + } +} + +TEST_CASE("randomly_flipped_nodes preserves structure on graph with reversing edges", "[randomly_flipped_nodes]") { + bdsg::HashGraph graph; + handle_t h1 = graph.create_handle("GATT", 1); + handle_t h2 = graph.create_handle("ACA", 2); + handle_t h3 = graph.create_handle("CGAT", 3); + handle_t h4 = graph.create_handle("TCGAA", 4); + + // Forward edges + graph.create_edge(h1, h2); + graph.create_edge(h2, h3); + graph.create_edge(h3, h4); + // Reversing edge: 4 fwd -> 3 rev + graph.create_edge(h4, graph.flip(h3)); + + auto original_edges = canonical_edge_pairs(graph); + + default_random_engine gen(test_seed_source()); + for (int i = 0; i < 10; i++) { + auto flipped = randomly_flipped_nodes(graph, 0.5, gen); + + REQUIRE(flipped.get_node_count() == 4); + REQUIRE(flipped.get_edge_count() == 4); + + auto flipped_edges = canonical_edge_pairs(flipped); + REQUIRE(original_edges == flipped_edges); + } +} + +TEST_CASE("randomly_flipped_nodes preserves structure on graph with self-loops", "[randomly_flipped_nodes]") { + bdsg::HashGraph graph; + handle_t h1 = graph.create_handle("ACGT", 1); + handle_t h2 = graph.create_handle("TTCC", 2); + + graph.create_edge(h1, h2); + // Self-loop on h1: fwd -> fwd + graph.create_edge(h1, h1); + // Inverting self-loop on h2: fwd -> rev + graph.create_edge(h2, graph.flip(h2)); + + auto original_edges = canonical_edge_pairs(graph); + + default_random_engine gen(test_seed_source()); + for (int i = 0; i < 10; i++) { + auto flipped = randomly_flipped_nodes(graph, 0.5, gen); + + REQUIRE(flipped.get_node_count() == 2); + REQUIRE(flipped.get_edge_count() == 3); + + auto flipped_edges = canonical_edge_pairs(flipped); + REQUIRE(original_edges == flipped_edges); + } +} + +TEST_CASE("randomly_flipped_nodes preserves structure on random graphs", "[randomly_flipped_nodes]") { + for (int trial = 0; trial < 5; trial++) { + bdsg::HashGraph graph; + random_graph(100, 10, 10, &graph); + + auto original_edges = canonical_edge_pairs(graph); + + default_random_engine gen(test_seed_source()); + for (int i = 0; i < 5; i++) { + auto flipped = randomly_flipped_nodes(graph, 0.5, gen); + + REQUIRE(flipped.get_node_count() == graph.get_node_count()); + REQUIRE(flipped.get_edge_count() == graph.get_edge_count()); + + auto flipped_edges = canonical_edge_pairs(flipped); + REQUIRE(original_edges == flipped_edges); + } + } +} + +TEST_CASE("randomly_flipped_nodes preserves node IDs", "[randomly_flipped_nodes]") { + bdsg::HashGraph graph; + graph.create_handle("AAA", 5); + graph.create_handle("CCC", 10); + graph.create_handle("GGG", 15); + graph.create_edge(graph.get_handle(5), graph.get_handle(10)); + graph.create_edge(graph.get_handle(10), graph.get_handle(15)); + + default_random_engine gen(test_seed_source()); + auto flipped = randomly_flipped_nodes(graph, 0.5, gen); + + REQUIRE(flipped.has_node(5)); + REQUIRE(flipped.has_node(10)); + REQUIRE(flipped.has_node(15)); +} + +TEST_CASE("randomly_flipped_nodes actually flips node sequences when p=1", "[randomly_flipped_nodes]") { + bdsg::HashGraph graph; + handle_t h1 = graph.create_handle("AAAC", 1); // RC = GTTT + + default_random_engine gen(test_seed_source()); + auto flipped = randomly_flipped_nodes(graph, 1.0, gen); + + // The forward sequence should be the RC of the original + REQUIRE(flipped.get_sequence(flipped.get_handle(1)) == "GTTT"); +} + +} // namespace unittest +} // namespace vg diff --git a/src/unittest/snarl_decomposition_fuzzer.cpp b/src/unittest/snarl_decomposition_fuzzer.cpp new file mode 100644 index 0000000000..b97e867271 --- /dev/null +++ b/src/unittest/snarl_decomposition_fuzzer.cpp @@ -0,0 +1,448 @@ +#include "catch.hpp" +#include "../handle.hpp" +#include + +#include "support/snarl_decomposition_fuzzer.hpp" + +#include +#include + +namespace vg { +namespace unittest { + +using namespace std; +using ET = ReplaySnarlFinder::EventType; +using Event = ReplaySnarlFinder::Event; + +/// Capture all events emitted by a traverse_decomposition call into a vector. +static vector> capture_events(const HandleGraphSnarlFinder& finder) { + vector> result; + finder.traverse_decomposition( + [&](handle_t h) { result.push_back({ET::BEGIN_CHAIN, h}); }, + [&](handle_t h) { result.push_back({ET::END_CHAIN, h}); }, + [&](handle_t h) { result.push_back({ET::BEGIN_SNARL, h}); }, + [&](handle_t h) { result.push_back({ET::END_SNARL, h}); } + ); + return result; +} + +TEST_CASE("ReplaySnarlFinder replays events faithfully", "[snarl_decomposition_fuzzer]") { + // Build a small graph to get real handles + bdsg::HashGraph graph; + graph.create_handle("A", 10); + graph.create_handle("C", 12); + graph.create_handle("G", 15); + graph.create_handle("T", 20); + graph.create_handle("AA", 22); + + handle_t h10f = graph.get_handle(10, false); + handle_t h12r = graph.get_handle(12, true); + handle_t h15r = graph.get_handle(15, true); + handle_t h20f = graph.get_handle(20, false); + handle_t h22f = graph.get_handle(22, false); + + vector events = { + {ET::BEGIN_CHAIN, h10f}, + {ET::BEGIN_SNARL, h10f}, + {ET::BEGIN_CHAIN, h12r}, + {ET::END_CHAIN, h15r}, + {ET::END_SNARL, h20f}, + {ET::BEGIN_SNARL, h20f}, + {ET::END_SNARL, h22f}, + {ET::END_CHAIN, h22f}, + }; + + ReplaySnarlFinder finder(events); + auto captured = capture_events(finder); + + REQUIRE(captured.size() == events.size()); + for (size_t i = 0; i < events.size(); i++) { + REQUIRE(captured[i].first == events[i].type); + REQUIRE(captured[i].second == events[i].handle); + } +} + +TEST_CASE("SnarlDecompositionFuzzer passes through when nothing is flipped", "[snarl_decomposition_fuzzer]") { + bdsg::HashGraph graph; + graph.create_handle("A", 10); + graph.create_handle("C", 12); + graph.create_handle("G", 15); + graph.create_handle("T", 20); + graph.create_handle("AA", 22); + + handle_t h10f = graph.get_handle(10, false); + handle_t h12r = graph.get_handle(12, true); + handle_t h15r = graph.get_handle(15, true); + handle_t h20f = graph.get_handle(20, false); + handle_t h22f = graph.get_handle(22, false); + + vector events = { + {ET::BEGIN_CHAIN, h10f}, + {ET::BEGIN_SNARL, h10f}, + {ET::BEGIN_CHAIN, h12r}, + {ET::END_CHAIN, h15r}, + {ET::END_SNARL, h20f}, + {ET::BEGIN_SNARL, h20f}, + {ET::END_SNARL, h22f}, + {ET::END_CHAIN, h22f}, + }; + + ReplaySnarlFinder replay(events); + + // No chains to flip + set> no_flips; + SnarlDecompositionFuzzer fuzzer(&graph, &replay, no_flips); + + auto captured = capture_events(fuzzer); + + REQUIRE(captured.size() == events.size()); + for (size_t i = 0; i < events.size(); i++) { + REQUIRE(captured[i].first == events[i].type); + REQUIRE(captured[i].second == events[i].handle); + } +} + +TEST_CASE("SnarlDecompositionFuzzer flips an outer chain", "[snarl_decomposition_fuzzer]") { + // Graph: + // Chain: 10fwd -> snarl(10fwd, 20fwd) -> snarl(20fwd, 22fwd) -> 22fwd + // Inside first snarl: chain 12rev->15rev + bdsg::HashGraph graph; + graph.create_handle("A", 10); + graph.create_handle("C", 12); + graph.create_handle("G", 15); + graph.create_handle("T", 20); + graph.create_handle("AA", 22); + + handle_t h10f = graph.get_handle(10, false); + handle_t h10r = graph.get_handle(10, true); + handle_t h12r = graph.get_handle(12, true); + handle_t h12f = graph.get_handle(12, false); + handle_t h15r = graph.get_handle(15, true); + handle_t h15f = graph.get_handle(15, false); + handle_t h20f = graph.get_handle(20, false); + handle_t h20r = graph.get_handle(20, true); + handle_t h22f = graph.get_handle(22, false); + handle_t h22r = graph.get_handle(22, true); + + vector events = { + {ET::BEGIN_CHAIN, h10f}, + {ET::BEGIN_SNARL, h10f}, + {ET::BEGIN_CHAIN, h12r}, + {ET::END_CHAIN, h15r}, + {ET::END_SNARL, h20f}, + {ET::BEGIN_SNARL, h20f}, + {ET::END_SNARL, h22f}, + {ET::END_CHAIN, h22f}, + }; + + ReplaySnarlFinder replay(events); + + SECTION("flip outer chain only") { + // Flip the outer chain (10fwd -> 22fwd) + set> flips = {{h10f, h22f}}; + SnarlDecompositionFuzzer fuzzer(&graph, &replay, flips); + + auto captured = capture_events(fuzzer); + + // Expected after flipping the outer chain: + // begin_chain(22 rev) + // begin_snarl(22 rev) -- was end_snarl(22 fwd), flipped + // end_snarl(20 rev) -- was begin_snarl(20 fwd), flipped + // begin_snarl(20 rev) -- was end_snarl(20 fwd), flipped + // begin_chain(12 rev) -- NOT flipped (not in flip set) + // end_chain(15 rev) + // end_snarl(10 rev) -- was begin_snarl(10 fwd), flipped + // end_chain(10 rev) + + vector> expected = { + {ET::BEGIN_CHAIN, h22r}, + {ET::BEGIN_SNARL, h22r}, + {ET::END_SNARL, h20r}, + {ET::BEGIN_SNARL, h20r}, + {ET::BEGIN_CHAIN, h12r}, + {ET::END_CHAIN, h15r}, + {ET::END_SNARL, h10r}, + {ET::END_CHAIN, h10r}, + }; + + REQUIRE(captured.size() == expected.size()); + for (size_t i = 0; i < expected.size(); i++) { + REQUIRE(captured[i].first == expected[i].first); + REQUIRE(captured[i].second == expected[i].second); + } + } + + SECTION("flip outer and nested chain") { + // Flip outer chain (10fwd->22fwd) AND nested chain (12rev->15rev) + set> flips = {{h10f, h22f}, {h12r, h15r}}; + SnarlDecompositionFuzzer fuzzer(&graph, &replay, flips); + + auto captured = capture_events(fuzzer); + + // Expected: outer chain flipped, AND the nested chain is also flipped + // The nested chain 12rev->15rev becomes: begin_chain(15fwd), end_chain(12fwd) + vector> expected = { + {ET::BEGIN_CHAIN, h22r}, + {ET::BEGIN_SNARL, h22r}, + {ET::END_SNARL, h20r}, + {ET::BEGIN_SNARL, h20r}, + {ET::BEGIN_CHAIN, h15f}, + {ET::END_CHAIN, h12f}, + {ET::END_SNARL, h10r}, + {ET::END_CHAIN, h10r}, + }; + + REQUIRE(captured.size() == expected.size()); + for (size_t i = 0; i < expected.size(); i++) { + REQUIRE(captured[i].first == expected[i].first); + REQUIRE(captured[i].second == expected[i].second); + } + } + + SECTION("flip nested chain only") { + // Flip only the nested chain (12rev->15rev), outer stays + set> flips = {{h12r, h15r}}; + SnarlDecompositionFuzzer fuzzer(&graph, &replay, flips); + + auto captured = capture_events(fuzzer); + + // Outer chain not flipped, nested chain flipped + vector> expected = { + {ET::BEGIN_CHAIN, h10f}, + {ET::BEGIN_SNARL, h10f}, + {ET::BEGIN_CHAIN, h15f}, + {ET::END_CHAIN, h12f}, + {ET::END_SNARL, h20f}, + {ET::BEGIN_SNARL, h20f}, + {ET::END_SNARL, h22f}, + {ET::END_CHAIN, h22f}, + }; + + REQUIRE(captured.size() == expected.size()); + for (size_t i = 0; i < expected.size(); i++) { + REQUIRE(captured[i].first == expected[i].first); + REQUIRE(captured[i].second == expected[i].second); + } + } +} + +TEST_CASE("SnarlDecompositionFuzzer handles empty chain (no snarls)", "[snarl_decomposition_fuzzer]") { + bdsg::HashGraph graph; + graph.create_handle("ACGT", 5); + + handle_t h5f = graph.get_handle(5, false); + handle_t h5r = graph.get_handle(5, true); + + // An empty chain: begin and end with same handle, no snarls inside + vector events = { + {ET::BEGIN_CHAIN, h5f}, + {ET::END_CHAIN, h5f}, + }; + + ReplaySnarlFinder replay(events); + + SECTION("flipping an empty chain") { + set> flips = {{h5f, h5f}}; + SnarlDecompositionFuzzer fuzzer(&graph, &replay, flips); + + auto captured = capture_events(fuzzer); + + // Flipped: begin_chain(flip(5fwd)) = begin_chain(5rev) + // end_chain(flip(5fwd)) = end_chain(5rev) + vector> expected = { + {ET::BEGIN_CHAIN, h5r}, + {ET::END_CHAIN, h5r}, + }; + + REQUIRE(captured.size() == expected.size()); + for (size_t i = 0; i < expected.size(); i++) { + REQUIRE(captured[i].first == expected[i].first); + REQUIRE(captured[i].second == expected[i].second); + } + } +} + +TEST_CASE("SnarlDecompositionFuzzer handles multiple top-level chains", "[snarl_decomposition_fuzzer]") { + bdsg::HashGraph graph; + graph.create_handle("A", 1); + graph.create_handle("C", 2); + graph.create_handle("G", 3); + graph.create_handle("T", 4); + + handle_t h1f = graph.get_handle(1, false); + handle_t h1r = graph.get_handle(1, true); + handle_t h2f = graph.get_handle(2, false); + handle_t h2r = graph.get_handle(2, true); + handle_t h3f = graph.get_handle(3, false); + handle_t h3r = graph.get_handle(3, true); + handle_t h4f = graph.get_handle(4, false); + handle_t h4r = graph.get_handle(4, true); + + // Two top-level chains in the root snarl + vector events = { + // Chain 1: 1fwd -> snarl -> 2fwd + {ET::BEGIN_CHAIN, h1f}, + {ET::BEGIN_SNARL, h1f}, + {ET::END_SNARL, h2f}, + {ET::END_CHAIN, h2f}, + // Chain 2: 3fwd -> snarl -> 4fwd + {ET::BEGIN_CHAIN, h3f}, + {ET::BEGIN_SNARL, h3f}, + {ET::END_SNARL, h4f}, + {ET::END_CHAIN, h4f}, + }; + + ReplaySnarlFinder replay(events); + + SECTION("flip only first chain") { + set> flips = {{h1f, h2f}}; + SnarlDecompositionFuzzer fuzzer(&graph, &replay, flips); + + auto captured = capture_events(fuzzer); + + vector> expected = { + {ET::BEGIN_CHAIN, h2r}, + {ET::BEGIN_SNARL, h2r}, + {ET::END_SNARL, h1r}, + {ET::END_CHAIN, h1r}, + {ET::BEGIN_CHAIN, h3f}, + {ET::BEGIN_SNARL, h3f}, + {ET::END_SNARL, h4f}, + {ET::END_CHAIN, h4f}, + }; + + REQUIRE(captured.size() == expected.size()); + for (size_t i = 0; i < expected.size(); i++) { + REQUIRE(captured[i].first == expected[i].first); + REQUIRE(captured[i].second == expected[i].second); + } + } + + SECTION("flip both chains") { + set> flips = {{h1f, h2f}, {h3f, h4f}}; + SnarlDecompositionFuzzer fuzzer(&graph, &replay, flips); + + auto captured = capture_events(fuzzer); + + vector> expected = { + {ET::BEGIN_CHAIN, h2r}, + {ET::BEGIN_SNARL, h2r}, + {ET::END_SNARL, h1r}, + {ET::END_CHAIN, h1r}, + {ET::BEGIN_CHAIN, h4r}, + {ET::BEGIN_SNARL, h4r}, + {ET::END_SNARL, h3r}, + {ET::END_CHAIN, h3r}, + }; + + REQUIRE(captured.size() == expected.size()); + for (size_t i = 0; i < expected.size(); i++) { + REQUIRE(captured[i].first == expected[i].first); + REQUIRE(captured[i].second == expected[i].second); + } + } +} + +TEST_CASE("SnarlDecompositionFuzzer handles deeply nested chains", "[snarl_decomposition_fuzzer]") { + bdsg::HashGraph graph; + for (nid_t i = 1; i <= 8; i++) { + graph.create_handle("A", i); + } + + handle_t h1f = graph.get_handle(1, false); + handle_t h1r = graph.get_handle(1, true); + handle_t h2f = graph.get_handle(2, false); + handle_t h2r = graph.get_handle(2, true); + handle_t h3f = graph.get_handle(3, false); + handle_t h3r = graph.get_handle(3, true); + handle_t h4f = graph.get_handle(4, false); + handle_t h4r = graph.get_handle(4, true); + handle_t h5f = graph.get_handle(5, false); + handle_t h5r = graph.get_handle(5, true); + handle_t h6f = graph.get_handle(6, false); + handle_t h6r = graph.get_handle(6, true); + + // Outer chain: 1->6 + // Snarl(1,4) + // Inner chain: 2->3 + // Snarl(2,3) [leaf snarl, no children] + // Snarl(4,6) + // Inner chain: 5->5 [empty/trivial] + vector events = { + {ET::BEGIN_CHAIN, h1f}, + {ET::BEGIN_SNARL, h1f}, + {ET::BEGIN_CHAIN, h2f}, + {ET::BEGIN_SNARL, h2f}, + {ET::END_SNARL, h3f}, + {ET::END_CHAIN, h3f}, + {ET::END_SNARL, h4f}, + {ET::BEGIN_SNARL, h4f}, + {ET::BEGIN_CHAIN, h5f}, + {ET::END_CHAIN, h5f}, + {ET::END_SNARL, h6f}, + {ET::END_CHAIN, h6f}, + }; + + ReplaySnarlFinder replay(events); + + SECTION("flip outer chain only") { + set> flips = {{h1f, h6f}}; + SnarlDecompositionFuzzer fuzzer(&graph, &replay, flips); + + auto captured = capture_events(fuzzer); + + // Outer flipped: snarls reversed, but inner chains NOT flipped + vector> expected = { + {ET::BEGIN_CHAIN, h6r}, + {ET::BEGIN_SNARL, h6r}, + {ET::BEGIN_CHAIN, h5f}, + {ET::END_CHAIN, h5f}, + {ET::END_SNARL, h4r}, + {ET::BEGIN_SNARL, h4r}, + {ET::BEGIN_CHAIN, h2f}, + {ET::BEGIN_SNARL, h2f}, + {ET::END_SNARL, h3f}, + {ET::END_CHAIN, h3f}, + {ET::END_SNARL, h1r}, + {ET::END_CHAIN, h1r}, + }; + + REQUIRE(captured.size() == expected.size()); + for (size_t i = 0; i < expected.size(); i++) { + REQUIRE(captured[i].first == expected[i].first); + REQUIRE(captured[i].second == expected[i].second); + } + } + + SECTION("flip outer and inner chain 2->3") { + set> flips = {{h1f, h6f}, {h2f, h3f}}; + SnarlDecompositionFuzzer fuzzer(&graph, &replay, flips); + + auto captured = capture_events(fuzzer); + + // Outer flipped, inner chain 2->3 also flipped + vector> expected = { + {ET::BEGIN_CHAIN, h6r}, + {ET::BEGIN_SNARL, h6r}, + {ET::BEGIN_CHAIN, h5f}, + {ET::END_CHAIN, h5f}, + {ET::END_SNARL, h4r}, + {ET::BEGIN_SNARL, h4r}, + {ET::BEGIN_CHAIN, h3r}, + {ET::BEGIN_SNARL, h3r}, + {ET::END_SNARL, h2r}, + {ET::END_CHAIN, h2r}, + {ET::END_SNARL, h1r}, + {ET::END_CHAIN, h1r}, + }; + + REQUIRE(captured.size() == expected.size()); + for (size_t i = 0; i < expected.size(); i++) { + REQUIRE(captured[i].first == expected[i].first); + REQUIRE(captured[i].second == expected[i].second); + } + } +} + +} // namespace unittest +} // namespace vg diff --git a/src/unittest/support/randomly_flipped_nodes.hpp b/src/unittest/support/randomly_flipped_nodes.hpp new file mode 100644 index 0000000000..40b00bda26 --- /dev/null +++ b/src/unittest/support/randomly_flipped_nodes.hpp @@ -0,0 +1,83 @@ +#ifndef VG_UNITTEST_RANDOMLY_FLIPPED_NODES_HPP_INCLUDED +#define VG_UNITTEST_RANDOMLY_FLIPPED_NODES_HPP_INCLUDED + +/** + * \file randomly_flipped_nodes.hpp + * Utility for creating a copy of a HandleGraph with a random subset of nodes + * flipped in orientation. + */ + +#include +#include +#include "handle.hpp" + +namespace vg { +namespace unittest { + +/** + * Return a copy of the given graph with approximately p_flip fraction of its + * nodes reversed in their local forward orientation. When a node is flipped, + * its sequence is reverse-complemented and all edges that connected to its + * forward orientation now connect to its reverse orientation, and vice versa. + * + * The returned graph preserves node IDs. + */ +template +bdsg::HashGraph randomly_flipped_nodes(const HandleGraph& source, double p_flip, URNG& generator) { + bdsg::HashGraph result; + + std::uniform_real_distribution dist(0.0, 1.0); + + // Track which nodes get flipped + std::unordered_set flipped; + + // Copy all nodes, flipping some + source.for_each_handle([&](const handle_t& handle) { + nid_t id = source.get_id(handle); + if (dist(generator) < p_flip) { + // Flip this node: store its reverse complement sequence as forward + result.create_handle(source.get_sequence(source.flip(handle)), id); + flipped.insert(id); + } else { + // Keep this node as-is + result.create_handle(source.get_sequence(handle), id); + } + }); + + // Copy all edges, adjusting for flipped nodes. + // An edge (left, right) means: leave left in its orientation, enter right + // in its orientation. If we flipped a node, we need to toggle the + // orientation on that side of the edge. + source.for_each_edge([&](const edge_t& edge) { + handle_t left = edge.first; + handle_t right = edge.second; + + nid_t left_id = source.get_id(left); + bool left_is_reverse = source.get_is_reverse(left); + + nid_t right_id = source.get_id(right); + bool right_is_reverse = source.get_is_reverse(right); + + // If we flipped a node, toggle the orientation for that side + if (flipped.count(left_id)) { + left_is_reverse = !left_is_reverse; + } + if (flipped.count(right_id)) { + right_is_reverse = !right_is_reverse; + } + + result.create_edge( + result.get_handle(left_id, left_is_reverse), + result.get_handle(right_id, right_is_reverse) + ); + + return true; + }); + + return result; +} + +} // namespace unittest +} // namespace vg + +#endif diff --git a/src/unittest/support/snarl_decomposition_fuzzer.cpp b/src/unittest/support/snarl_decomposition_fuzzer.cpp new file mode 100644 index 0000000000..90379b8986 --- /dev/null +++ b/src/unittest/support/snarl_decomposition_fuzzer.cpp @@ -0,0 +1,274 @@ +#include "snarl_decomposition_fuzzer.hpp" + +#include + +namespace vg { +namespace unittest { + +using namespace std; + +// An event captured from the wrapped finder's decomposition +struct CapturedEvent { + enum Type { BEGIN_CHAIN, END_CHAIN, BEGIN_SNARL, END_SNARL }; + Type type; + handle_t handle; +}; + +/// Find the matching end event for a begin event at position `start`. +/// For BEGIN_CHAIN finds matching END_CHAIN; for BEGIN_SNARL finds END_SNARL. +/// Handles nesting correctly. +static size_t find_matching_end(const vector& events, size_t start) { + bool is_chain = (events[start].type == CapturedEvent::BEGIN_CHAIN); + CapturedEvent::Type begin_type = is_chain ? CapturedEvent::BEGIN_CHAIN : CapturedEvent::BEGIN_SNARL; + CapturedEvent::Type end_type = is_chain ? CapturedEvent::END_CHAIN : CapturedEvent::END_SNARL; + + int depth = 0; + for (size_t i = start; i < events.size(); i++) { + if (events[i].type == begin_type) { + depth++; + } else if (events[i].type == end_type) { + depth--; + if (depth == 0) { + return i; + } + } + } + assert(false); + return events.size(); +} + +// Forward declaration +static void process_chain( + const vector& events, + size_t chain_start, size_t chain_end, + const HandleGraph& graph, + const function& should_flip, + const function& begin_chain, + const function& end_chain, + const function& begin_snarl, + const function& end_snarl); + +/// Process events in the range [start, end), recursively applying chain flipping. +/// This handles top-level chains in the root snarl. +static void process_events( + const vector& events, + size_t start, size_t end, + const HandleGraph& graph, + const function& should_flip, + const function& begin_chain, + const function& end_chain, + const function& begin_snarl, + const function& end_snarl) +{ + size_t i = start; + while (i < end) { + if (events[i].type == CapturedEvent::BEGIN_CHAIN) { + size_t chain_end_idx = find_matching_end(events, i); + process_chain(events, i, chain_end_idx, graph, should_flip, + begin_chain, end_chain, begin_snarl, end_snarl); + i = chain_end_idx + 1; + } else { + switch (events[i].type) { + case CapturedEvent::BEGIN_SNARL: + begin_snarl(events[i].handle); + break; + case CapturedEvent::END_SNARL: + end_snarl(events[i].handle); + break; + default: + break; + } + i++; + } + } +} + +/// Process a snarl's interior (the events between BEGIN_SNARL and END_SNARL, +/// exclusive of those boundary events), recursively processing child chains. +static void process_snarl_interior( + const vector& events, + size_t interior_start, size_t interior_end, + const HandleGraph& graph, + const function& should_flip, + const function& begin_chain, + const function& end_chain, + const function& begin_snarl, + const function& end_snarl) +{ + size_t i = interior_start; + while (i < interior_end) { + if (events[i].type == CapturedEvent::BEGIN_CHAIN) { + size_t chain_end_idx = find_matching_end(events, i); + process_chain(events, i, chain_end_idx, graph, should_flip, + begin_chain, end_chain, begin_snarl, end_snarl); + i = chain_end_idx + 1; + } else { + i++; + } + } +} + +/// Process a single chain (events[chain_start] to events[chain_end], inclusive). +/// Decides whether to flip it, and recursively processes nested chains. +static void process_chain( + const vector& events, + size_t chain_start, size_t chain_end, + const HandleGraph& graph, + const function& should_flip, + const function& begin_chain, + const function& end_chain, + const function& begin_snarl, + const function& end_snarl) +{ + handle_t orig_begin = events[chain_start].handle; + handle_t orig_end = events[chain_end].handle; + + if (should_flip(orig_begin, orig_end)) { + // Flip this chain. + // Collect the snarls inside this chain. + struct SnarlSpan { + size_t begin_idx; // index of BEGIN_SNARL + size_t end_idx; // index of END_SNARL + }; + vector snarls; + + size_t i = chain_start + 1; + while (i < chain_end) { + if (events[i].type == CapturedEvent::BEGIN_SNARL) { + size_t snarl_end = find_matching_end(events, i); + snarls.push_back({i, snarl_end}); + i = snarl_end + 1; + } else { + i++; + } + } + + // Emit flipped chain: begin with flip(end), end with flip(begin) + begin_chain(graph.flip(orig_end)); + + // Emit snarls in reverse order with flipped boundaries + for (int s = (int)snarls.size() - 1; s >= 0; s--) { + const auto& snarl = snarls[s]; + handle_t snarl_begin_h = events[snarl.begin_idx].handle; + handle_t snarl_end_h = events[snarl.end_idx].handle; + + begin_snarl(graph.flip(snarl_end_h)); + + // Collect child chains inside this snarl + struct ChainSpan { + size_t begin_idx; + size_t end_idx; + }; + vector child_chains; + size_t j = snarl.begin_idx + 1; + while (j < snarl.end_idx) { + if (events[j].type == CapturedEvent::BEGIN_CHAIN) { + size_t nested_end = find_matching_end(events, j); + child_chains.push_back({j, nested_end}); + j = nested_end + 1; + } else { + j++; + } + } + + // Emit child chains in reverse order, recursively processing each + for (int c = (int)child_chains.size() - 1; c >= 0; c--) { + process_chain(events, child_chains[c].begin_idx, child_chains[c].end_idx, + graph, should_flip, begin_chain, end_chain, begin_snarl, end_snarl); + } + + end_snarl(graph.flip(snarl_begin_h)); + } + + end_chain(graph.flip(orig_begin)); + } else { + // Don't flip this chain, but still recursively process nested chains + begin_chain(orig_begin); + + // Walk through interior + size_t i = chain_start + 1; + while (i < chain_end) { + if (events[i].type == CapturedEvent::BEGIN_SNARL) { + size_t snarl_end = find_matching_end(events, i); + begin_snarl(events[i].handle); + + // Process snarl interior (child chains) + process_snarl_interior(events, i + 1, snarl_end, graph, should_flip, + begin_chain, end_chain, begin_snarl, end_snarl); + + end_snarl(events[snarl_end].handle); + i = snarl_end + 1; + } else { + i++; + } + } + + end_chain(orig_end); + } +} + +// SnarlDecompositionFuzzer deterministic constructor +SnarlDecompositionFuzzer::SnarlDecompositionFuzzer( + const HandleGraph* graph, + const HandleGraphSnarlFinder* finder, + const set>& chains_to_flip) + : HandleGraphSnarlFinder(graph), wrapped(finder) +{ + should_flip = [chains_to_flip](handle_t begin, handle_t end) -> bool { + return chains_to_flip.count({begin, end}) > 0; + }; +} + +void SnarlDecompositionFuzzer::traverse_decomposition( + const function& begin_chain, + const function& end_chain, + const function& begin_snarl, + const function& end_snarl) const +{ + // Step 1: Capture all events from the wrapped finder + vector events; + wrapped->traverse_decomposition( + [&](handle_t h) { events.push_back({CapturedEvent::BEGIN_CHAIN, h}); }, + [&](handle_t h) { events.push_back({CapturedEvent::END_CHAIN, h}); }, + [&](handle_t h) { events.push_back({CapturedEvent::BEGIN_SNARL, h}); }, + [&](handle_t h) { events.push_back({CapturedEvent::END_SNARL, h}); } + ); + + // Step 2: Process events, flipping chains as needed + process_events(events, 0, events.size(), *graph, should_flip, + begin_chain, end_chain, begin_snarl, end_snarl); +} + +// ReplaySnarlFinder implementation + +ReplaySnarlFinder::ReplaySnarlFinder(const vector& events) + : HandleGraphSnarlFinder(nullptr), events(events) +{ +} + +void ReplaySnarlFinder::traverse_decomposition( + const function& begin_chain, + const function& end_chain, + const function& begin_snarl, + const function& end_snarl) const +{ + for (const auto& event : events) { + switch (event.type) { + case EventType::BEGIN_CHAIN: + begin_chain(event.handle); + break; + case EventType::END_CHAIN: + end_chain(event.handle); + break; + case EventType::BEGIN_SNARL: + begin_snarl(event.handle); + break; + case EventType::END_SNARL: + end_snarl(event.handle); + break; + } + } +} + +} // namespace unittest +} // namespace vg diff --git a/src/unittest/support/snarl_decomposition_fuzzer.hpp b/src/unittest/support/snarl_decomposition_fuzzer.hpp new file mode 100644 index 0000000000..be4ca61cd4 --- /dev/null +++ b/src/unittest/support/snarl_decomposition_fuzzer.hpp @@ -0,0 +1,139 @@ +#ifndef VG_UNITTEST_SNARL_DECOMPOSITION_FUZZER_HPP_INCLUDED +#define VG_UNITTEST_SNARL_DECOMPOSITION_FUZZER_HPP_INCLUDED + +/** + * \file snarl_decomposition_fuzzer.hpp + * Provides SnarlDecompositionFuzzer, which wraps a HandleGraphSnarlFinder and + * randomly flips chains in the snarl decomposition, and ReplaySnarlFinder, + * which replays a scripted sequence of decomposition events. + */ + +#include +#include +#include +#include +#include +#include "snarls.hpp" +#include "handle.hpp" + +namespace vg { +namespace unittest { + +/** + * A HandleGraphSnarlFinder that wraps another HandleGraphSnarlFinder and + * randomly flips chains in the snarl decomposition. Each chain in a snarl + * has an independent probability p_flip of being emitted in the reverse + * order and orientation. Flipping is applied recursively: nested chains + * inside a flipped chain also have the same probability of being flipped. + * + * For deterministic testing, chains_to_flip can be provided, which is a set + * of (begin_handle, end_handle) pairs identifying chains to flip. When + * provided, p_flip and the generator are ignored. + */ +class SnarlDecompositionFuzzer : public HandleGraphSnarlFinder { +public: + /** + * Construct a fuzzer wrapping the given finder, flipping chains with + * probability p_flip using the given random generator. + * The graph pointer is needed to look up the graph for the base class. + */ + template + SnarlDecompositionFuzzer(const HandleGraph* graph, + const HandleGraphSnarlFinder* finder, + double p_flip, URNG& generator); + + /** + * Construct a fuzzer wrapping the given finder, flipping exactly the + * chains identified by the given set of (begin_handle, end_handle) pairs. + * The handles should be the inward-facing begin and outward-facing end + * handles as originally emitted by the wrapped finder. + */ + SnarlDecompositionFuzzer(const HandleGraph* graph, + const HandleGraphSnarlFinder* finder, + const std::set>& chains_to_flip); + + virtual ~SnarlDecompositionFuzzer() = default; + + /** + * Traverse the snarl decomposition, flipping selected chains. + */ + virtual void traverse_decomposition( + const std::function& begin_chain, + const std::function& end_chain, + const std::function& begin_snarl, + const std::function& end_snarl + ) const override; + +private: + /// The wrapped snarl finder + const HandleGraphSnarlFinder* wrapped; + + /// Function that decides whether to flip a chain given its begin and end handles + std::function should_flip; +}; + +/** + * A HandleGraphSnarlFinder that replays a scripted sequence of decomposition + * events. Useful for testing SnarlDecompositionFuzzer without needing a real + * graph or snarl finder. + */ +class ReplaySnarlFinder : public HandleGraphSnarlFinder { +public: + /// Event types for the decomposition + enum class EventType { + BEGIN_CHAIN, + END_CHAIN, + BEGIN_SNARL, + END_SNARL + }; + + /// An event in the decomposition + struct Event { + EventType type; + handle_t handle; + }; + + /** + * Construct a replay finder that will emit the given events. + * The graph pointer can be null since we never actually use it. + */ + ReplaySnarlFinder(const std::vector& events); + + virtual ~ReplaySnarlFinder() = default; + + /** + * Replay the scripted events. + */ + virtual void traverse_decomposition( + const std::function& begin_chain, + const std::function& end_chain, + const std::function& begin_snarl, + const std::function& end_snarl + ) const override; + +private: + std::vector events; +}; + +// Template implementation + +template +SnarlDecompositionFuzzer::SnarlDecompositionFuzzer( + const HandleGraph* graph, + const HandleGraphSnarlFinder* finder, + double p_flip, URNG& generator) + : HandleGraphSnarlFinder(graph), wrapped(finder) +{ + // Capture generator state by making a copy of the engine + // We need a shared_ptr because the lambda must be copyable + auto gen = std::make_shared(generator); + auto dist = std::make_shared>(0.0, 1.0); + should_flip = [gen, dist, p_flip](handle_t, handle_t) -> bool { + return (*dist)(*gen) < p_flip; + }; +} + +} // namespace unittest +} // namespace vg + +#endif From c2f3279c46d2ab6755734d205964c081fa853507 Mon Sep 17 00:00:00 2001 From: Adam Novak Date: Fri, 13 Feb 2026 16:04:17 -0500 Subject: [PATCH 33/77] Synthesize code that puts child chains the right way around --- src/unittest/snarl_decomposition_fuzzer.cpp | 60 ++++++----- .../support/snarl_decomposition_fuzzer.cpp | 100 ++++++++++-------- 2 files changed, 86 insertions(+), 74 deletions(-) diff --git a/src/unittest/snarl_decomposition_fuzzer.cpp b/src/unittest/snarl_decomposition_fuzzer.cpp index b97e867271..dbb7dfee25 100644 --- a/src/unittest/snarl_decomposition_fuzzer.cpp +++ b/src/unittest/snarl_decomposition_fuzzer.cpp @@ -145,22 +145,16 @@ TEST_CASE("SnarlDecompositionFuzzer flips an outer chain", "[snarl_decomposition auto captured = capture_events(fuzzer); // Expected after flipping the outer chain: - // begin_chain(22 rev) - // begin_snarl(22 rev) -- was end_snarl(22 fwd), flipped - // end_snarl(20 rev) -- was begin_snarl(20 fwd), flipped - // begin_snarl(20 rev) -- was end_snarl(20 fwd), flipped - // begin_chain(12 rev) -- NOT flipped (not in flip set) - // end_chain(15 rev) - // end_snarl(10 rev) -- was begin_snarl(10 fwd), flipped - // end_chain(10 rev) - + // Flipping a chain reverses everything inside it, including children. + // The nested chain 12rev->15rev gets reversed to 15fwd->12fwd as + // part of the parent flip. vector> expected = { {ET::BEGIN_CHAIN, h22r}, {ET::BEGIN_SNARL, h22r}, {ET::END_SNARL, h20r}, {ET::BEGIN_SNARL, h20r}, - {ET::BEGIN_CHAIN, h12r}, - {ET::END_CHAIN, h15r}, + {ET::BEGIN_CHAIN, h15f}, + {ET::END_CHAIN, h12f}, {ET::END_SNARL, h10r}, {ET::END_CHAIN, h10r}, }; @@ -179,15 +173,16 @@ TEST_CASE("SnarlDecompositionFuzzer flips an outer chain", "[snarl_decomposition auto captured = capture_events(fuzzer); - // Expected: outer chain flipped, AND the nested chain is also flipped - // The nested chain 12rev->15rev becomes: begin_chain(15fwd), end_chain(12fwd) + // Expected: outer chain flipped (reversing everything, including + // the nested chain to 15fwd->12fwd), AND THEN the nested chain is + // flipped again back to its original orientation 12rev->15rev. vector> expected = { {ET::BEGIN_CHAIN, h22r}, {ET::BEGIN_SNARL, h22r}, {ET::END_SNARL, h20r}, {ET::BEGIN_SNARL, h20r}, - {ET::BEGIN_CHAIN, h15f}, - {ET::END_CHAIN, h12f}, + {ET::BEGIN_CHAIN, h12r}, + {ET::END_CHAIN, h15r}, {ET::END_SNARL, h10r}, {ET::END_CHAIN, h10r}, }; @@ -391,18 +386,22 @@ TEST_CASE("SnarlDecompositionFuzzer handles deeply nested chains", "[snarl_decom auto captured = capture_events(fuzzer); - // Outer flipped: snarls reversed, but inner chains NOT flipped + // Outer flipped: snarls reversed, and inner chains also reversed + // as part of the parent flip. + // Chain 5f->5f reversed to 5r->5r. + // Chain 2f->3f (containing snarl 2f->3f) reversed to 3r->2r + // (containing snarl 3r->2r). vector> expected = { {ET::BEGIN_CHAIN, h6r}, {ET::BEGIN_SNARL, h6r}, - {ET::BEGIN_CHAIN, h5f}, - {ET::END_CHAIN, h5f}, + {ET::BEGIN_CHAIN, h5r}, + {ET::END_CHAIN, h5r}, {ET::END_SNARL, h4r}, {ET::BEGIN_SNARL, h4r}, - {ET::BEGIN_CHAIN, h2f}, - {ET::BEGIN_SNARL, h2f}, - {ET::END_SNARL, h3f}, - {ET::END_CHAIN, h3f}, + {ET::BEGIN_CHAIN, h3r}, + {ET::BEGIN_SNARL, h3r}, + {ET::END_SNARL, h2r}, + {ET::END_CHAIN, h2r}, {ET::END_SNARL, h1r}, {ET::END_CHAIN, h1r}, }; @@ -420,18 +419,21 @@ TEST_CASE("SnarlDecompositionFuzzer handles deeply nested chains", "[snarl_decom auto captured = capture_events(fuzzer); - // Outer flipped, inner chain 2->3 also flipped + // Outer flipped (reversing everything, including chain 2f->3f to + // 3r->2r and chain 5f->5f to 5r->5r), AND THEN inner chain 2f->3f + // is flipped again back to its original orientation 2f->3f. + // Chain 5f->5f is NOT in flip set, so it stays reversed as 5r->5r. vector> expected = { {ET::BEGIN_CHAIN, h6r}, {ET::BEGIN_SNARL, h6r}, - {ET::BEGIN_CHAIN, h5f}, - {ET::END_CHAIN, h5f}, + {ET::BEGIN_CHAIN, h5r}, + {ET::END_CHAIN, h5r}, {ET::END_SNARL, h4r}, {ET::BEGIN_SNARL, h4r}, - {ET::BEGIN_CHAIN, h3r}, - {ET::BEGIN_SNARL, h3r}, - {ET::END_SNARL, h2r}, - {ET::END_CHAIN, h2r}, + {ET::BEGIN_CHAIN, h2f}, + {ET::BEGIN_SNARL, h2f}, + {ET::END_SNARL, h3f}, + {ET::END_CHAIN, h3f}, {ET::END_SNARL, h1r}, {ET::END_CHAIN, h1r}, }; diff --git a/src/unittest/support/snarl_decomposition_fuzzer.cpp b/src/unittest/support/snarl_decomposition_fuzzer.cpp index 90379b8986..0b7f6cd839 100644 --- a/src/unittest/support/snarl_decomposition_fuzzer.cpp +++ b/src/unittest/support/snarl_decomposition_fuzzer.cpp @@ -37,12 +37,17 @@ static size_t find_matching_end(const vector& events, size_t star return events.size(); } -// Forward declaration +// Forward declaration. +// do_flip: whether this chain should be emitted reversed. A chain's effective +// flip is the XOR of all flip decisions from itself and its ancestors: +// flipping a parent flips all its descendants, and if a descendant is also +// in the flip set it gets flipped again (canceling the parent's flip). static void process_chain( const vector& events, size_t chain_start, size_t chain_end, const HandleGraph& graph, const function& should_flip, + bool do_flip, const function& begin_chain, const function& end_chain, const function& begin_snarl, @@ -64,8 +69,11 @@ static void process_events( while (i < end) { if (events[i].type == CapturedEvent::BEGIN_CHAIN) { size_t chain_end_idx = find_matching_end(events, i); + handle_t orig_begin = events[i].handle; + handle_t orig_end = events[chain_end_idx].handle; + bool do_flip = should_flip(orig_begin, orig_end); process_chain(events, i, chain_end_idx, graph, should_flip, - begin_chain, end_chain, begin_snarl, end_snarl); + do_flip, begin_chain, end_chain, begin_snarl, end_snarl); i = chain_end_idx + 1; } else { switch (events[i].type) { @@ -83,38 +91,45 @@ static void process_events( } } -/// Process a snarl's interior (the events between BEGIN_SNARL and END_SNARL, -/// exclusive of those boundary events), recursively processing child chains. -static void process_snarl_interior( +/// Collect the index spans of child chains within a snarl's interior +/// (between interior_start and interior_end, exclusive of snarl boundaries). +static vector> collect_child_chains( const vector& events, - size_t interior_start, size_t interior_end, - const HandleGraph& graph, - const function& should_flip, - const function& begin_chain, - const function& end_chain, - const function& begin_snarl, - const function& end_snarl) + size_t interior_start, size_t interior_end) { + vector> children; size_t i = interior_start; while (i < interior_end) { if (events[i].type == CapturedEvent::BEGIN_CHAIN) { size_t chain_end_idx = find_matching_end(events, i); - process_chain(events, i, chain_end_idx, graph, should_flip, - begin_chain, end_chain, begin_snarl, end_snarl); + children.push_back({i, chain_end_idx}); i = chain_end_idx + 1; } else { i++; } } + return children; +} + +/// Compute the effective flip for a child chain: the XOR of the parent's +/// effective flip and the child's own should_flip decision (always evaluated +/// on the child's original handles from the event stream). +static bool child_effective_flip( + bool parent_do_flip, + const function& should_flip, + handle_t child_orig_begin, handle_t child_orig_end) +{ + return parent_do_flip != should_flip(child_orig_begin, child_orig_end); } /// Process a single chain (events[chain_start] to events[chain_end], inclusive). -/// Decides whether to flip it, and recursively processes nested chains. +/// do_flip indicates whether this chain should be emitted reversed. static void process_chain( const vector& events, size_t chain_start, size_t chain_end, const HandleGraph& graph, const function& should_flip, + bool do_flip, const function& begin_chain, const function& end_chain, const function& begin_snarl, @@ -123,12 +138,14 @@ static void process_chain( handle_t orig_begin = events[chain_start].handle; handle_t orig_end = events[chain_end].handle; - if (should_flip(orig_begin, orig_end)) { - // Flip this chain. + if (do_flip) { + // Flip this chain: reverse snarl order, flip all boundary handles, + // and recursively process children with their effective flip state. + // Collect the snarls inside this chain. struct SnarlSpan { - size_t begin_idx; // index of BEGIN_SNARL - size_t end_idx; // index of END_SNARL + size_t begin_idx; + size_t end_idx; }; vector snarls; @@ -143,7 +160,6 @@ static void process_chain( } } - // Emit flipped chain: begin with flip(end), end with flip(begin) begin_chain(graph.flip(orig_end)); // Emit snarls in reverse order with flipped boundaries @@ -154,27 +170,15 @@ static void process_chain( begin_snarl(graph.flip(snarl_end_h)); - // Collect child chains inside this snarl - struct ChainSpan { - size_t begin_idx; - size_t end_idx; - }; - vector child_chains; - size_t j = snarl.begin_idx + 1; - while (j < snarl.end_idx) { - if (events[j].type == CapturedEvent::BEGIN_CHAIN) { - size_t nested_end = find_matching_end(events, j); - child_chains.push_back({j, nested_end}); - j = nested_end + 1; - } else { - j++; - } - } - - // Emit child chains in reverse order, recursively processing each - for (int c = (int)child_chains.size() - 1; c >= 0; c--) { - process_chain(events, child_chains[c].begin_idx, child_chains[c].end_idx, - graph, should_flip, begin_chain, end_chain, begin_snarl, end_snarl); + // Collect and emit child chains in reverse order + auto children = collect_child_chains(events, snarl.begin_idx + 1, snarl.end_idx); + for (int c = (int)children.size() - 1; c >= 0; c--) { + handle_t child_begin = events[children[c].first].handle; + handle_t child_end = events[children[c].second].handle; + bool child_flip = child_effective_flip(do_flip, should_flip, child_begin, child_end); + process_chain(events, children[c].first, children[c].second, + graph, should_flip, child_flip, + begin_chain, end_chain, begin_snarl, end_snarl); } end_snarl(graph.flip(snarl_begin_h)); @@ -183,18 +187,24 @@ static void process_chain( end_chain(graph.flip(orig_begin)); } else { // Don't flip this chain, but still recursively process nested chains + // which may have their own flip decisions. begin_chain(orig_begin); - // Walk through interior size_t i = chain_start + 1; while (i < chain_end) { if (events[i].type == CapturedEvent::BEGIN_SNARL) { size_t snarl_end = find_matching_end(events, i); begin_snarl(events[i].handle); - // Process snarl interior (child chains) - process_snarl_interior(events, i + 1, snarl_end, graph, should_flip, - begin_chain, end_chain, begin_snarl, end_snarl); + auto children = collect_child_chains(events, i + 1, snarl_end); + for (auto& child : children) { + handle_t child_begin = events[child.first].handle; + handle_t child_end = events[child.second].handle; + bool child_flip = child_effective_flip(do_flip, should_flip, child_begin, child_end); + process_chain(events, child.first, child.second, + graph, should_flip, child_flip, + begin_chain, end_chain, begin_snarl, end_snarl); + } end_snarl(events[snarl_end].handle); i = snarl_end + 1; From c3cd3b7b8031aeb0eefa7ced5bce4ff927c094e7 Mon Sep 17 00:00:00 2001 From: Adam Novak Date: Fri, 13 Feb 2026 16:45:47 -0500 Subject: [PATCH 34/77] Synthesize much simpler code that I designed myself because stochastically selecting your data structures considered harmful --- .../support/snarl_decomposition_fuzzer.cpp | 340 +++++++----------- .../support/snarl_decomposition_fuzzer.hpp | 42 +-- 2 files changed, 146 insertions(+), 236 deletions(-) diff --git a/src/unittest/support/snarl_decomposition_fuzzer.cpp b/src/unittest/support/snarl_decomposition_fuzzer.cpp index 0b7f6cd839..384de2ec86 100644 --- a/src/unittest/support/snarl_decomposition_fuzzer.cpp +++ b/src/unittest/support/snarl_decomposition_fuzzer.cpp @@ -1,254 +1,162 @@ #include "snarl_decomposition_fuzzer.hpp" #include +#include namespace vg { namespace unittest { using namespace std; +using ET = DecompositionEventType; -// An event captured from the wrapped finder's decomposition -struct CapturedEvent { - enum Type { BEGIN_CHAIN, END_CHAIN, BEGIN_SNARL, END_SNARL }; - Type type; - handle_t handle; -}; - -/// Find the matching end event for a begin event at position `start`. -/// For BEGIN_CHAIN finds matching END_CHAIN; for BEGIN_SNARL finds END_SNARL. -/// Handles nesting correctly. -static size_t find_matching_end(const vector& events, size_t start) { - bool is_chain = (events[start].type == CapturedEvent::BEGIN_CHAIN); - CapturedEvent::Type begin_type = is_chain ? CapturedEvent::BEGIN_CHAIN : CapturedEvent::BEGIN_SNARL; - CapturedEvent::Type end_type = is_chain ? CapturedEvent::END_CHAIN : CapturedEvent::END_SNARL; - - int depth = 0; - for (size_t i = start; i < events.size(); i++) { - if (events[i].type == begin_type) { - depth++; - } else if (events[i].type == end_type) { - depth--; - if (depth == 0) { - return i; - } - } - } - assert(false); - return events.size(); +// SnarlDecompositionFuzzer deterministic constructor +SnarlDecompositionFuzzer::SnarlDecompositionFuzzer( + const HandleGraph* graph, + const HandleGraphSnarlFinder* finder, + const set>& chains_to_flip) + : HandleGraphSnarlFinder(graph), wrapped(finder) +{ + should_flip = [chains_to_flip](handle_t begin, handle_t end) -> bool { + return chains_to_flip.count({begin, end}) > 0; + }; } -// Forward declaration. -// do_flip: whether this chain should be emitted reversed. A chain's effective -// flip is the XOR of all flip decisions from itself and its ancestors: -// flipping a parent flips all its descendants, and if a descendant is also -// in the flip set it gets flipped again (canceling the parent's flip). -static void process_chain( - const vector& events, - size_t chain_start, size_t chain_end, - const HandleGraph& graph, - const function& should_flip, - bool do_flip, - const function& begin_chain, - const function& end_chain, - const function& begin_snarl, - const function& end_snarl); - -/// Process events in the range [start, end), recursively applying chain flipping. -/// This handles top-level chains in the root snarl. -static void process_events( - const vector& events, - size_t start, size_t end, +/// Emit an event, transforming it based on direction. +/// Forward: emit as-is. +/// Backward: swap begin/end types and flip handles. +static void emit_event( + const DecompositionEvent& event, + bool forward, const HandleGraph& graph, - const function& should_flip, const function& begin_chain, const function& end_chain, const function& begin_snarl, const function& end_snarl) { - size_t i = start; - while (i < end) { - if (events[i].type == CapturedEvent::BEGIN_CHAIN) { - size_t chain_end_idx = find_matching_end(events, i); - handle_t orig_begin = events[i].handle; - handle_t orig_end = events[chain_end_idx].handle; - bool do_flip = should_flip(orig_begin, orig_end); - process_chain(events, i, chain_end_idx, graph, should_flip, - do_flip, begin_chain, end_chain, begin_snarl, end_snarl); - i = chain_end_idx + 1; - } else { - switch (events[i].type) { - case CapturedEvent::BEGIN_SNARL: - begin_snarl(events[i].handle); - break; - case CapturedEvent::END_SNARL: - end_snarl(events[i].handle); - break; - default: - break; - } - i++; + if (forward) { + switch (event.type) { + case ET::BEGIN_CHAIN: begin_chain(event.handle); break; + case ET::END_CHAIN: end_chain(event.handle); break; + case ET::BEGIN_SNARL: begin_snarl(event.handle); break; + case ET::END_SNARL: end_snarl(event.handle); break; } - } -} - -/// Collect the index spans of child chains within a snarl's interior -/// (between interior_start and interior_end, exclusive of snarl boundaries). -static vector> collect_child_chains( - const vector& events, - size_t interior_start, size_t interior_end) -{ - vector> children; - size_t i = interior_start; - while (i < interior_end) { - if (events[i].type == CapturedEvent::BEGIN_CHAIN) { - size_t chain_end_idx = find_matching_end(events, i); - children.push_back({i, chain_end_idx}); - i = chain_end_idx + 1; - } else { - i++; + } else { + handle_t flipped = graph.flip(event.handle); + switch (event.type) { + case ET::BEGIN_CHAIN: end_chain(flipped); break; + case ET::END_CHAIN: begin_chain(flipped); break; + case ET::BEGIN_SNARL: end_snarl(flipped); break; + case ET::END_SNARL: begin_snarl(flipped); break; } } - return children; } -/// Compute the effective flip for a child chain: the XOR of the parent's -/// effective flip and the child's own should_flip decision (always evaluated -/// on the child's original handles from the event stream). -static bool child_effective_flip( - bool parent_do_flip, - const function& should_flip, - handle_t child_orig_begin, handle_t child_orig_end) -{ - return parent_do_flip != should_flip(child_orig_begin, child_orig_end); -} - -/// Process a single chain (events[chain_start] to events[chain_end], inclusive). -/// do_flip indicates whether this chain should be emitted reversed. -static void process_chain( - const vector& events, - size_t chain_start, size_t chain_end, - const HandleGraph& graph, - const function& should_flip, - bool do_flip, +void SnarlDecompositionFuzzer::traverse_decomposition( const function& begin_chain, const function& end_chain, const function& begin_snarl, - const function& end_snarl) + const function& end_snarl) const { - handle_t orig_begin = events[chain_start].handle; - handle_t orig_end = events[chain_end].handle; - - if (do_flip) { - // Flip this chain: reverse snarl order, flip all boundary handles, - // and recursively process children with their effective flip state. + // Step 1: Capture all events from the wrapped finder. + vector events; + wrapped->traverse_decomposition( + [&](handle_t h) { events.push_back({ET::BEGIN_CHAIN, h}); }, + [&](handle_t h) { events.push_back({ET::END_CHAIN, h}); }, + [&](handle_t h) { events.push_back({ET::BEGIN_SNARL, h}); }, + [&](handle_t h) { events.push_back({ET::END_SNARL, h}); } + ); - // Collect the snarls inside this chain. - struct SnarlSpan { - size_t begin_idx; - size_t end_idx; - }; - vector snarls; + if (events.empty()) return; - size_t i = chain_start + 1; - while (i < chain_end) { - if (events[i].type == CapturedEvent::BEGIN_SNARL) { - size_t snarl_end = find_matching_end(events, i); - snarls.push_back({i, snarl_end}); - i = snarl_end + 1; - } else { - i++; + // Step 2: Build pairing vector mapping each begin to its matching end + // and vice versa, using separate stacks for chains and snarls. + size_t n = events.size(); + vector pair_of(n); + { + stack chain_stack, snarl_stack; + for (size_t i = 0; i < n; i++) { + switch (events[i].type) { + case ET::BEGIN_CHAIN: + chain_stack.push(i); + break; + case ET::END_CHAIN: + assert(!chain_stack.empty()); + pair_of[i] = chain_stack.top(); + pair_of[chain_stack.top()] = i; + chain_stack.pop(); + break; + case ET::BEGIN_SNARL: + snarl_stack.push(i); + break; + case ET::END_SNARL: + assert(!snarl_stack.empty()); + pair_of[i] = snarl_stack.top(); + pair_of[snarl_stack.top()] = i; + snarl_stack.pop(); + break; } } + } - begin_chain(graph.flip(orig_end)); - - // Emit snarls in reverse order with flipped boundaries - for (int s = (int)snarls.size() - 1; s >= 0; s--) { - const auto& snarl = snarls[s]; - handle_t snarl_begin_h = events[snarl.begin_idx].handle; - handle_t snarl_end_h = events[snarl.end_idx].handle; - - begin_snarl(graph.flip(snarl_end_h)); - - // Collect and emit child chains in reverse order - auto children = collect_child_chains(events, snarl.begin_idx + 1, snarl.end_idx); - for (int c = (int)children.size() - 1; c >= 0; c--) { - handle_t child_begin = events[children[c].first].handle; - handle_t child_end = events[children[c].second].handle; - bool child_flip = child_effective_flip(do_flip, should_flip, child_begin, child_end); - process_chain(events, children[c].first, children[c].second, - graph, should_flip, child_flip, - begin_chain, end_chain, begin_snarl, end_snarl); - } - - end_snarl(graph.flip(snarl_begin_h)); + // Step 3: Walk through events with a cursor, flipping chains as needed. + // When we flip a chain, we jump to the other end and reverse direction, + // pushing the entry point onto a stack. When the cursor reaches a stack + // entry point, we jump back to the far end and restore direction. + struct FlipEntry { + size_t entry_index; + bool original_forward; + }; + stack flip_stack; + + int64_t cursor = 0; + bool forward = true; + + while (cursor >= 0 && cursor < (int64_t)n) { + size_t idx = (size_t)cursor; + + // Check if we've returned to the entry point of a flipped chain. + if (!flip_stack.empty() && idx == flip_stack.top().entry_index) { + emit_event(events[idx], forward, *graph, + begin_chain, end_chain, begin_snarl, end_snarl); + FlipEntry entry = flip_stack.top(); + flip_stack.pop(); + cursor = (int64_t)pair_of[entry.entry_index]; + forward = entry.original_forward; + cursor += forward ? 1 : -1; + continue; } - end_chain(graph.flip(orig_begin)); - } else { - // Don't flip this chain, but still recursively process nested chains - // which may have their own flip decisions. - begin_chain(orig_begin); - - size_t i = chain_start + 1; - while (i < chain_end) { - if (events[i].type == CapturedEvent::BEGIN_SNARL) { - size_t snarl_end = find_matching_end(events, i); - begin_snarl(events[i].handle); - - auto children = collect_child_chains(events, i + 1, snarl_end); - for (auto& child : children) { - handle_t child_begin = events[child.first].handle; - handle_t child_end = events[child.second].handle; - bool child_flip = child_effective_flip(do_flip, should_flip, child_begin, child_end); - process_chain(events, child.first, child.second, - graph, should_flip, child_flip, - begin_chain, end_chain, begin_snarl, end_snarl); - } - - end_snarl(events[snarl_end].handle); - i = snarl_end + 1; - } else { - i++; + // Check if we're entering a chain. + bool is_chain_entry = + (forward && events[idx].type == ET::BEGIN_CHAIN) || + (!forward && events[idx].type == ET::END_CHAIN); + + if (is_chain_entry) { + size_t begin_idx = forward ? idx : pair_of[idx]; + size_t end_idx = forward ? pair_of[idx] : idx; + handle_t begin_handle = events[begin_idx].handle; + handle_t end_handle = events[end_idx].handle; + + if (should_flip(begin_handle, end_handle)) { + // Flip: remember where we entered, jump to the other end, + // reverse direction, emit the entry event there. + flip_stack.push({idx, forward}); + cursor = (int64_t)pair_of[idx]; + forward = !forward; + emit_event(events[(size_t)cursor], forward, *graph, + begin_chain, end_chain, begin_snarl, end_snarl); + cursor += forward ? 1 : -1; + continue; } } - end_chain(orig_end); + // Normal event: emit and advance. + emit_event(events[idx], forward, *graph, + begin_chain, end_chain, begin_snarl, end_snarl); + cursor += forward ? 1 : -1; } } -// SnarlDecompositionFuzzer deterministic constructor -SnarlDecompositionFuzzer::SnarlDecompositionFuzzer( - const HandleGraph* graph, - const HandleGraphSnarlFinder* finder, - const set>& chains_to_flip) - : HandleGraphSnarlFinder(graph), wrapped(finder) -{ - should_flip = [chains_to_flip](handle_t begin, handle_t end) -> bool { - return chains_to_flip.count({begin, end}) > 0; - }; -} - -void SnarlDecompositionFuzzer::traverse_decomposition( - const function& begin_chain, - const function& end_chain, - const function& begin_snarl, - const function& end_snarl) const -{ - // Step 1: Capture all events from the wrapped finder - vector events; - wrapped->traverse_decomposition( - [&](handle_t h) { events.push_back({CapturedEvent::BEGIN_CHAIN, h}); }, - [&](handle_t h) { events.push_back({CapturedEvent::END_CHAIN, h}); }, - [&](handle_t h) { events.push_back({CapturedEvent::BEGIN_SNARL, h}); }, - [&](handle_t h) { events.push_back({CapturedEvent::END_SNARL, h}); } - ); - - // Step 2: Process events, flipping chains as needed - process_events(events, 0, events.size(), *graph, should_flip, - begin_chain, end_chain, begin_snarl, end_snarl); -} - // ReplaySnarlFinder implementation ReplaySnarlFinder::ReplaySnarlFinder(const vector& events) @@ -264,16 +172,16 @@ void ReplaySnarlFinder::traverse_decomposition( { for (const auto& event : events) { switch (event.type) { - case EventType::BEGIN_CHAIN: + case ET::BEGIN_CHAIN: begin_chain(event.handle); break; - case EventType::END_CHAIN: + case ET::END_CHAIN: end_chain(event.handle); break; - case EventType::BEGIN_SNARL: + case ET::BEGIN_SNARL: begin_snarl(event.handle); break; - case EventType::END_SNARL: + case ET::END_SNARL: end_snarl(event.handle); break; } diff --git a/src/unittest/support/snarl_decomposition_fuzzer.hpp b/src/unittest/support/snarl_decomposition_fuzzer.hpp index be4ca61cd4..467115b8c3 100644 --- a/src/unittest/support/snarl_decomposition_fuzzer.hpp +++ b/src/unittest/support/snarl_decomposition_fuzzer.hpp @@ -19,12 +19,26 @@ namespace vg { namespace unittest { +/// Event types for snarl decomposition traversal. +enum class DecompositionEventType { + BEGIN_CHAIN, + END_CHAIN, + BEGIN_SNARL, + END_SNARL +}; + +/// A single event in a snarl decomposition traversal. +struct DecompositionEvent { + DecompositionEventType type; + handle_t handle; +}; + /** * A HandleGraphSnarlFinder that wraps another HandleGraphSnarlFinder and - * randomly flips chains in the snarl decomposition. Each chain in a snarl - * has an independent probability p_flip of being emitted in the reverse - * order and orientation. Flipping is applied recursively: nested chains - * inside a flipped chain also have the same probability of being flipped. + * randomly flips chains in the snarl decomposition. Flipping a chain reverses + * the entire chain including all children; if a child chain is also selected + * for flipping, it gets flipped again (canceling the parent's flip for that + * child). * * For deterministic testing, chains_to_flip can be provided, which is a set * of (begin_handle, end_handle) pairs identifying chains to flip. When @@ -35,7 +49,7 @@ class SnarlDecompositionFuzzer : public HandleGraphSnarlFinder { /** * Construct a fuzzer wrapping the given finder, flipping chains with * probability p_flip using the given random generator. - * The graph pointer is needed to look up the graph for the base class. + * The graph pointer is needed to flip handles. */ template SnarlDecompositionFuzzer(const HandleGraph* graph, @@ -79,19 +93,9 @@ class SnarlDecompositionFuzzer : public HandleGraphSnarlFinder { */ class ReplaySnarlFinder : public HandleGraphSnarlFinder { public: - /// Event types for the decomposition - enum class EventType { - BEGIN_CHAIN, - END_CHAIN, - BEGIN_SNARL, - END_SNARL - }; - - /// An event in the decomposition - struct Event { - EventType type; - handle_t handle; - }; + /// Alias for shared event types + using EventType = DecompositionEventType; + using Event = DecompositionEvent; /** * Construct a replay finder that will emit the given events. @@ -124,8 +128,6 @@ SnarlDecompositionFuzzer::SnarlDecompositionFuzzer( double p_flip, URNG& generator) : HandleGraphSnarlFinder(graph), wrapped(finder) { - // Capture generator state by making a copy of the engine - // We need a shared_ptr because the lambda must be copyable auto gen = std::make_shared(generator); auto dist = std::make_shared>(0.0, 1.0); should_flip = [gen, dist, p_flip](handle_t, handle_t) -> bool { From c75e7ed80875ddc4673aa2f4b00188a7ffefe707 Mon Sep 17 00:00:00 2001 From: Adam Novak Date: Fri, 13 Feb 2026 16:53:32 -0500 Subject: [PATCH 35/77] Synthesize slightly more encapsulated code --- .../support/snarl_decomposition_fuzzer.cpp | 16 ++++++---------- .../support/snarl_decomposition_fuzzer.hpp | 18 ++++++++++++++---- 2 files changed, 20 insertions(+), 14 deletions(-) diff --git a/src/unittest/support/snarl_decomposition_fuzzer.cpp b/src/unittest/support/snarl_decomposition_fuzzer.cpp index 384de2ec86..165fa15f9c 100644 --- a/src/unittest/support/snarl_decomposition_fuzzer.cpp +++ b/src/unittest/support/snarl_decomposition_fuzzer.cpp @@ -21,17 +21,13 @@ SnarlDecompositionFuzzer::SnarlDecompositionFuzzer( }; } -/// Emit an event, transforming it based on direction. -/// Forward: emit as-is. -/// Backward: swap begin/end types and flip handles. -static void emit_event( +void SnarlDecompositionFuzzer::emit_event( const DecompositionEvent& event, bool forward, - const HandleGraph& graph, const function& begin_chain, const function& end_chain, const function& begin_snarl, - const function& end_snarl) + const function& end_snarl) const { if (forward) { switch (event.type) { @@ -41,7 +37,7 @@ static void emit_event( case ET::END_SNARL: end_snarl(event.handle); break; } } else { - handle_t flipped = graph.flip(event.handle); + handle_t flipped = graph->flip(event.handle); switch (event.type) { case ET::BEGIN_CHAIN: end_chain(flipped); break; case ET::END_CHAIN: begin_chain(flipped); break; @@ -116,7 +112,7 @@ void SnarlDecompositionFuzzer::traverse_decomposition( // Check if we've returned to the entry point of a flipped chain. if (!flip_stack.empty() && idx == flip_stack.top().entry_index) { - emit_event(events[idx], forward, *graph, + emit_event(events[idx], forward, begin_chain, end_chain, begin_snarl, end_snarl); FlipEntry entry = flip_stack.top(); flip_stack.pop(); @@ -143,7 +139,7 @@ void SnarlDecompositionFuzzer::traverse_decomposition( flip_stack.push({idx, forward}); cursor = (int64_t)pair_of[idx]; forward = !forward; - emit_event(events[(size_t)cursor], forward, *graph, + emit_event(events[(size_t)cursor], forward, begin_chain, end_chain, begin_snarl, end_snarl); cursor += forward ? 1 : -1; continue; @@ -151,7 +147,7 @@ void SnarlDecompositionFuzzer::traverse_decomposition( } // Normal event: emit and advance. - emit_event(events[idx], forward, *graph, + emit_event(events[idx], forward, begin_chain, end_chain, begin_snarl, end_snarl); cursor += forward ? 1 : -1; } diff --git a/src/unittest/support/snarl_decomposition_fuzzer.hpp b/src/unittest/support/snarl_decomposition_fuzzer.hpp index 467115b8c3..2ea28a8e18 100644 --- a/src/unittest/support/snarl_decomposition_fuzzer.hpp +++ b/src/unittest/support/snarl_decomposition_fuzzer.hpp @@ -84,6 +84,18 @@ class SnarlDecompositionFuzzer : public HandleGraphSnarlFinder { /// Function that decides whether to flip a chain given its begin and end handles std::function should_flip; + + /// Emit an event, transforming it based on direction. + /// Forward: emit as-is. + /// Backward: swap begin/end types and flip handles. + void emit_event( + const DecompositionEvent& event, + bool forward, + const std::function& begin_chain, + const std::function& end_chain, + const std::function& begin_snarl, + const std::function& end_snarl + ) const; }; /** @@ -128,10 +140,8 @@ SnarlDecompositionFuzzer::SnarlDecompositionFuzzer( double p_flip, URNG& generator) : HandleGraphSnarlFinder(graph), wrapped(finder) { - auto gen = std::make_shared(generator); - auto dist = std::make_shared>(0.0, 1.0); - should_flip = [gen, dist, p_flip](handle_t, handle_t) -> bool { - return (*dist)(*gen) < p_flip; + should_flip = [&generator, p_flip](handle_t, handle_t) -> bool { + return std::uniform_real_distribution(0.0, 1.0)(generator) < p_flip; }; } From a9ba894cfbb4abcf13dde4cb6d61ce65b2946bc4 Mon Sep 17 00:00:00 2001 From: Adam Novak Date: Fri, 13 Feb 2026 18:10:55 -0500 Subject: [PATCH 36/77] Simplify the cursor loop and the flipping determination --- src/unittest/randomly_flipped_nodes.cpp | 2 - src/unittest/snarl_decomposition_fuzzer.cpp | 54 ++++--- .../support/snarl_decomposition_fuzzer.cpp | 146 ++++++++---------- .../support/snarl_decomposition_fuzzer.hpp | 48 +++--- 4 files changed, 114 insertions(+), 136 deletions(-) diff --git a/src/unittest/randomly_flipped_nodes.cpp b/src/unittest/randomly_flipped_nodes.cpp index 0c1d158d24..6950432bc9 100644 --- a/src/unittest/randomly_flipped_nodes.cpp +++ b/src/unittest/randomly_flipped_nodes.cpp @@ -13,8 +13,6 @@ namespace vg { namespace unittest { -using namespace std; - /// Get the canonicalized set of edge sequence pairs from a graph. /// Each edge is represented as a pair of sequences (left_seq, right_seq) read /// in the orientation of the edge. To canonicalize, we compare each pair diff --git a/src/unittest/snarl_decomposition_fuzzer.cpp b/src/unittest/snarl_decomposition_fuzzer.cpp index dbb7dfee25..16a0ad6212 100644 --- a/src/unittest/snarl_decomposition_fuzzer.cpp +++ b/src/unittest/snarl_decomposition_fuzzer.cpp @@ -5,18 +5,17 @@ #include "support/snarl_decomposition_fuzzer.hpp" #include -#include +#include namespace vg { namespace unittest { -using namespace std; using ET = ReplaySnarlFinder::EventType; using Event = ReplaySnarlFinder::Event; /// Capture all events emitted by a traverse_decomposition call into a vector. -static vector> capture_events(const HandleGraphSnarlFinder& finder) { - vector> result; +static std::vector> capture_events(const HandleGraphSnarlFinder& finder) { + std::vector> result; finder.traverse_decomposition( [&](handle_t h) { result.push_back({ET::BEGIN_CHAIN, h}); }, [&](handle_t h) { result.push_back({ET::END_CHAIN, h}); }, @@ -41,7 +40,7 @@ TEST_CASE("ReplaySnarlFinder replays events faithfully", "[snarl_decomposition_f handle_t h20f = graph.get_handle(20, false); handle_t h22f = graph.get_handle(22, false); - vector events = { + std::vector events = { {ET::BEGIN_CHAIN, h10f}, {ET::BEGIN_SNARL, h10f}, {ET::BEGIN_CHAIN, h12r}, @@ -76,7 +75,7 @@ TEST_CASE("SnarlDecompositionFuzzer passes through when nothing is flipped", "[s handle_t h20f = graph.get_handle(20, false); handle_t h22f = graph.get_handle(22, false); - vector events = { + std::vector events = { {ET::BEGIN_CHAIN, h10f}, {ET::BEGIN_SNARL, h10f}, {ET::BEGIN_CHAIN, h12r}, @@ -90,8 +89,7 @@ TEST_CASE("SnarlDecompositionFuzzer passes through when nothing is flipped", "[s ReplaySnarlFinder replay(events); // No chains to flip - set> no_flips; - SnarlDecompositionFuzzer fuzzer(&graph, &replay, no_flips); + SnarlDecompositionFuzzer fuzzer(&graph, &replay, {}); auto captured = capture_events(fuzzer); @@ -124,7 +122,7 @@ TEST_CASE("SnarlDecompositionFuzzer flips an outer chain", "[snarl_decomposition handle_t h22f = graph.get_handle(22, false); handle_t h22r = graph.get_handle(22, true); - vector events = { + std::vector events = { {ET::BEGIN_CHAIN, h10f}, {ET::BEGIN_SNARL, h10f}, {ET::BEGIN_CHAIN, h12r}, @@ -139,7 +137,7 @@ TEST_CASE("SnarlDecompositionFuzzer flips an outer chain", "[snarl_decomposition SECTION("flip outer chain only") { // Flip the outer chain (10fwd -> 22fwd) - set> flips = {{h10f, h22f}}; + std::unordered_set flips {10, 22}; SnarlDecompositionFuzzer fuzzer(&graph, &replay, flips); auto captured = capture_events(fuzzer); @@ -148,7 +146,7 @@ TEST_CASE("SnarlDecompositionFuzzer flips an outer chain", "[snarl_decomposition // Flipping a chain reverses everything inside it, including children. // The nested chain 12rev->15rev gets reversed to 15fwd->12fwd as // part of the parent flip. - vector> expected = { + std::vector> expected = { {ET::BEGIN_CHAIN, h22r}, {ET::BEGIN_SNARL, h22r}, {ET::END_SNARL, h20r}, @@ -168,7 +166,7 @@ TEST_CASE("SnarlDecompositionFuzzer flips an outer chain", "[snarl_decomposition SECTION("flip outer and nested chain") { // Flip outer chain (10fwd->22fwd) AND nested chain (12rev->15rev) - set> flips = {{h10f, h22f}, {h12r, h15r}}; + std::unordered_set flips {10, 22, 12, 15}; SnarlDecompositionFuzzer fuzzer(&graph, &replay, flips); auto captured = capture_events(fuzzer); @@ -176,7 +174,7 @@ TEST_CASE("SnarlDecompositionFuzzer flips an outer chain", "[snarl_decomposition // Expected: outer chain flipped (reversing everything, including // the nested chain to 15fwd->12fwd), AND THEN the nested chain is // flipped again back to its original orientation 12rev->15rev. - vector> expected = { + std::vector> expected = { {ET::BEGIN_CHAIN, h22r}, {ET::BEGIN_SNARL, h22r}, {ET::END_SNARL, h20r}, @@ -196,13 +194,13 @@ TEST_CASE("SnarlDecompositionFuzzer flips an outer chain", "[snarl_decomposition SECTION("flip nested chain only") { // Flip only the nested chain (12rev->15rev), outer stays - set> flips = {{h12r, h15r}}; + std::unordered_set flips {12, 15}; SnarlDecompositionFuzzer fuzzer(&graph, &replay, flips); auto captured = capture_events(fuzzer); // Outer chain not flipped, nested chain flipped - vector> expected = { + std::vector> expected = { {ET::BEGIN_CHAIN, h10f}, {ET::BEGIN_SNARL, h10f}, {ET::BEGIN_CHAIN, h15f}, @@ -229,7 +227,7 @@ TEST_CASE("SnarlDecompositionFuzzer handles empty chain (no snarls)", "[snarl_de handle_t h5r = graph.get_handle(5, true); // An empty chain: begin and end with same handle, no snarls inside - vector events = { + std::vector events = { {ET::BEGIN_CHAIN, h5f}, {ET::END_CHAIN, h5f}, }; @@ -237,14 +235,14 @@ TEST_CASE("SnarlDecompositionFuzzer handles empty chain (no snarls)", "[snarl_de ReplaySnarlFinder replay(events); SECTION("flipping an empty chain") { - set> flips = {{h5f, h5f}}; + std::unordered_set flips {5}; SnarlDecompositionFuzzer fuzzer(&graph, &replay, flips); auto captured = capture_events(fuzzer); // Flipped: begin_chain(flip(5fwd)) = begin_chain(5rev) // end_chain(flip(5fwd)) = end_chain(5rev) - vector> expected = { + std::vector> expected = { {ET::BEGIN_CHAIN, h5r}, {ET::END_CHAIN, h5r}, }; @@ -274,7 +272,7 @@ TEST_CASE("SnarlDecompositionFuzzer handles multiple top-level chains", "[snarl_ handle_t h4r = graph.get_handle(4, true); // Two top-level chains in the root snarl - vector events = { + std::vector events = { // Chain 1: 1fwd -> snarl -> 2fwd {ET::BEGIN_CHAIN, h1f}, {ET::BEGIN_SNARL, h1f}, @@ -290,12 +288,12 @@ TEST_CASE("SnarlDecompositionFuzzer handles multiple top-level chains", "[snarl_ ReplaySnarlFinder replay(events); SECTION("flip only first chain") { - set> flips = {{h1f, h2f}}; + std::unordered_set flips {1, 2}; SnarlDecompositionFuzzer fuzzer(&graph, &replay, flips); auto captured = capture_events(fuzzer); - vector> expected = { + std::vector> expected = { {ET::BEGIN_CHAIN, h2r}, {ET::BEGIN_SNARL, h2r}, {ET::END_SNARL, h1r}, @@ -314,12 +312,12 @@ TEST_CASE("SnarlDecompositionFuzzer handles multiple top-level chains", "[snarl_ } SECTION("flip both chains") { - set> flips = {{h1f, h2f}, {h3f, h4f}}; + std::unordered_set flips {1, 2, 3, 4}; SnarlDecompositionFuzzer fuzzer(&graph, &replay, flips); auto captured = capture_events(fuzzer); - vector> expected = { + std::vector> expected = { {ET::BEGIN_CHAIN, h2r}, {ET::BEGIN_SNARL, h2r}, {ET::END_SNARL, h1r}, @@ -363,7 +361,7 @@ TEST_CASE("SnarlDecompositionFuzzer handles deeply nested chains", "[snarl_decom // Snarl(2,3) [leaf snarl, no children] // Snarl(4,6) // Inner chain: 5->5 [empty/trivial] - vector events = { + std::vector events = { {ET::BEGIN_CHAIN, h1f}, {ET::BEGIN_SNARL, h1f}, {ET::BEGIN_CHAIN, h2f}, @@ -381,7 +379,7 @@ TEST_CASE("SnarlDecompositionFuzzer handles deeply nested chains", "[snarl_decom ReplaySnarlFinder replay(events); SECTION("flip outer chain only") { - set> flips = {{h1f, h6f}}; + std::unordered_set flips {1, 6}; SnarlDecompositionFuzzer fuzzer(&graph, &replay, flips); auto captured = capture_events(fuzzer); @@ -391,7 +389,7 @@ TEST_CASE("SnarlDecompositionFuzzer handles deeply nested chains", "[snarl_decom // Chain 5f->5f reversed to 5r->5r. // Chain 2f->3f (containing snarl 2f->3f) reversed to 3r->2r // (containing snarl 3r->2r). - vector> expected = { + std::vector> expected = { {ET::BEGIN_CHAIN, h6r}, {ET::BEGIN_SNARL, h6r}, {ET::BEGIN_CHAIN, h5r}, @@ -414,7 +412,7 @@ TEST_CASE("SnarlDecompositionFuzzer handles deeply nested chains", "[snarl_decom } SECTION("flip outer and inner chain 2->3") { - set> flips = {{h1f, h6f}, {h2f, h3f}}; + std::unordered_set flips {1, 6, 2, 3}; SnarlDecompositionFuzzer fuzzer(&graph, &replay, flips); auto captured = capture_events(fuzzer); @@ -423,7 +421,7 @@ TEST_CASE("SnarlDecompositionFuzzer handles deeply nested chains", "[snarl_decom // 3r->2r and chain 5f->5f to 5r->5r), AND THEN inner chain 2f->3f // is flipped again back to its original orientation 2f->3f. // Chain 5f->5f is NOT in flip set, so it stays reversed as 5r->5r. - vector> expected = { + std::vector> expected = { {ET::BEGIN_CHAIN, h6r}, {ET::BEGIN_SNARL, h6r}, {ET::BEGIN_CHAIN, h5r}, diff --git a/src/unittest/support/snarl_decomposition_fuzzer.cpp b/src/unittest/support/snarl_decomposition_fuzzer.cpp index 165fa15f9c..4a3ca3583c 100644 --- a/src/unittest/support/snarl_decomposition_fuzzer.cpp +++ b/src/unittest/support/snarl_decomposition_fuzzer.cpp @@ -6,55 +6,45 @@ namespace vg { namespace unittest { -using namespace std; using ET = DecompositionEventType; // SnarlDecompositionFuzzer deterministic constructor SnarlDecompositionFuzzer::SnarlDecompositionFuzzer( const HandleGraph* graph, const HandleGraphSnarlFinder* finder, - const set>& chains_to_flip) + const std::unordered_set& chains_to_flip) : HandleGraphSnarlFinder(graph), wrapped(finder) { - should_flip = [chains_to_flip](handle_t begin, handle_t end) -> bool { - return chains_to_flip.count({begin, end}) > 0; + + should_flip = [chains_to_flip, graph](nid_t node_id) -> bool { + return chains_to_flip.count(node_id); }; } -void SnarlDecompositionFuzzer::emit_event( - const DecompositionEvent& event, - bool forward, +void SnarlDecompositionFuzzer::traverse_decomposition( const function& begin_chain, const function& end_chain, const function& begin_snarl, const function& end_snarl) const { - if (forward) { - switch (event.type) { - case ET::BEGIN_CHAIN: begin_chain(event.handle); break; - case ET::END_CHAIN: end_chain(event.handle); break; - case ET::BEGIN_SNARL: begin_snarl(event.handle); break; - case ET::END_SNARL: end_snarl(event.handle); break; + // This vector is indexable with the event types. + std::vector*> handlers {&begin_chain, &end_chain, &begin_snarl, &end_snarl}; + + // Make a helper to emit an event, transforming it based on direction. + // Forward: emit as-is. + // Backward: swap begin/end types and flip handles. + std::function emit_event = [&](const DecompositionEvent& event, bool reverse) { + if (reverse) { + // Flip the event around if needed + emit_event(flip(event, graph), false); + } else { + // Call the right handler on the event's handle. + (*handlers.at((int)event.type))(event.handle); } - } else { - handle_t flipped = graph->flip(event.handle); - switch (event.type) { - case ET::BEGIN_CHAIN: end_chain(flipped); break; - case ET::END_CHAIN: begin_chain(flipped); break; - case ET::BEGIN_SNARL: end_snarl(flipped); break; - case ET::END_SNARL: begin_snarl(flipped); break; - } - } -} + }; -void SnarlDecompositionFuzzer::traverse_decomposition( - const function& begin_chain, - const function& end_chain, - const function& begin_snarl, - const function& end_snarl) const -{ // Step 1: Capture all events from the wrapped finder. - vector events; + std::vector events; wrapped->traverse_decomposition( [&](handle_t h) { events.push_back({ET::BEGIN_CHAIN, h}); }, [&](handle_t h) { events.push_back({ET::END_CHAIN, h}); }, @@ -62,15 +52,16 @@ void SnarlDecompositionFuzzer::traverse_decomposition( [&](handle_t h) { events.push_back({ET::END_SNARL, h}); } ); - if (events.empty()) return; + if (events.empty()) { + return; + } // Step 2: Build pairing vector mapping each begin to its matching end // and vice versa, using separate stacks for chains and snarls. - size_t n = events.size(); - vector pair_of(n); + std::vector pair_of(events.size()); { stack chain_stack, snarl_stack; - for (size_t i = 0; i < n; i++) { + for (size_t i = 0; i < events.size(); i++) { switch (events[i].type) { case ET::BEGIN_CHAIN: chain_stack.push(i); @@ -100,71 +91,56 @@ void SnarlDecompositionFuzzer::traverse_decomposition( // entry point, we jump back to the far end and restore direction. struct FlipEntry { size_t entry_index; - bool original_forward; + bool original_reverse; }; - stack flip_stack; - - int64_t cursor = 0; - bool forward = true; - - while (cursor >= 0 && cursor < (int64_t)n) { - size_t idx = (size_t)cursor; - - // Check if we've returned to the entry point of a flipped chain. - if (!flip_stack.empty() && idx == flip_stack.top().entry_index) { - emit_event(events[idx], forward, - begin_chain, end_chain, begin_snarl, end_snarl); + std::stack flip_stack; + + bool reverse = false; + for (size_t cursor = 0; cursor != events.size(); cursor += reverse ? -1 : 1) { + // We know if we're entering a chain, we can't be at a stack pop point. + // So we can handle those cases separately. + + if (events[cursor].type == (reverse ? ET::END_CHAIN : ET::BEGIN_CHAIN) && + should_flip(graph->get_id(events[cursor].handle))) { + + // We're entering a chain, and this is a chain we want to flip. So + // flip before emitting anything. + + // Flip: remember where we entered, jump to the other end, + // reverse direction, emit the entry event there. + flip_stack.push({cursor, reverse}); + cursor = pair_of[cursor]; + reverse = !reverse; + } + + // Emit the event here + emit_event(events[cursor], reverse); + + if (!flip_stack.empty() && cursor == flip_stack.top().entry_index) { + // We've returned to the entry point of a flipped chain, so after + // emitting, go back to the entry orientation and jump to the other + // side, so we can advance out of it. + FlipEntry entry = flip_stack.top(); flip_stack.pop(); - cursor = (int64_t)pair_of[entry.entry_index]; - forward = entry.original_forward; - cursor += forward ? 1 : -1; - continue; + cursor = pair_of[entry.entry_index]; + reverse = entry.original_reverse; } - - // Check if we're entering a chain. - bool is_chain_entry = - (forward && events[idx].type == ET::BEGIN_CHAIN) || - (!forward && events[idx].type == ET::END_CHAIN); - - if (is_chain_entry) { - size_t begin_idx = forward ? idx : pair_of[idx]; - size_t end_idx = forward ? pair_of[idx] : idx; - handle_t begin_handle = events[begin_idx].handle; - handle_t end_handle = events[end_idx].handle; - - if (should_flip(begin_handle, end_handle)) { - // Flip: remember where we entered, jump to the other end, - // reverse direction, emit the entry event there. - flip_stack.push({idx, forward}); - cursor = (int64_t)pair_of[idx]; - forward = !forward; - emit_event(events[(size_t)cursor], forward, - begin_chain, end_chain, begin_snarl, end_snarl); - cursor += forward ? 1 : -1; - continue; - } - } - - // Normal event: emit and advance. - emit_event(events[idx], forward, - begin_chain, end_chain, begin_snarl, end_snarl); - cursor += forward ? 1 : -1; } } // ReplaySnarlFinder implementation -ReplaySnarlFinder::ReplaySnarlFinder(const vector& events) +ReplaySnarlFinder::ReplaySnarlFinder(const std::vector& events) : HandleGraphSnarlFinder(nullptr), events(events) { } void ReplaySnarlFinder::traverse_decomposition( - const function& begin_chain, - const function& end_chain, - const function& begin_snarl, - const function& end_snarl) const + const std::function& begin_chain, + const std::function& end_chain, + const std::function& begin_snarl, + const std::function& end_snarl) const { for (const auto& event : events) { switch (event.type) { diff --git a/src/unittest/support/snarl_decomposition_fuzzer.hpp b/src/unittest/support/snarl_decomposition_fuzzer.hpp index 2ea28a8e18..ee741db472 100644 --- a/src/unittest/support/snarl_decomposition_fuzzer.hpp +++ b/src/unittest/support/snarl_decomposition_fuzzer.hpp @@ -21,18 +21,29 @@ namespace unittest { /// Event types for snarl decomposition traversal. enum class DecompositionEventType { - BEGIN_CHAIN, + BEGIN_CHAIN = 0, END_CHAIN, BEGIN_SNARL, END_SNARL }; +/// Flip the polatiry of an event type (start vs. end) +inline DecompositionEventType flip(const DecompositionEventType& t) { + // We can flip by toggling the low bit. + return (DecompositionEventType)((int) t ^ 1); +} + /// A single event in a snarl decomposition traversal. struct DecompositionEvent { DecompositionEventType type; handle_t handle; }; +/// Flip the polarity of a whole event (event type between begin and end, and handle orientation) +inline DecompositionEvent flip(const DecompositionEvent& e, const HandleGraph* g) { + return {flip(e.type), g->flip(e.handle)}; +} + /** * A HandleGraphSnarlFinder that wraps another HandleGraphSnarlFinder and * randomly flips chains in the snarl decomposition. Flipping a chain reverses @@ -57,14 +68,20 @@ class SnarlDecompositionFuzzer : public HandleGraphSnarlFinder { double p_flip, URNG& generator); /** - * Construct a fuzzer wrapping the given finder, flipping exactly the - * chains identified by the given set of (begin_handle, end_handle) pairs. - * The handles should be the inward-facing begin and outward-facing end - * handles as originally emitted by the wrapped finder. + * Construct a fuzzer wrapping the given finder, flipping the chains + * bounded by the given node IDs. + * + * You should provide both bounding IDs for each chain, but only the one + * that the chain is actually arrived at through during the traversal will + * really get used. + * + * Note that a node can bound at most one chain. + * + * This is mostly for testing the fuzzer itself. */ SnarlDecompositionFuzzer(const HandleGraph* graph, const HandleGraphSnarlFinder* finder, - const std::set>& chains_to_flip); + const std::unordered_set& chains_to_flip); virtual ~SnarlDecompositionFuzzer() = default; @@ -82,20 +99,9 @@ class SnarlDecompositionFuzzer : public HandleGraphSnarlFinder { /// The wrapped snarl finder const HandleGraphSnarlFinder* wrapped; - /// Function that decides whether to flip a chain given its begin and end handles - std::function should_flip; - - /// Emit an event, transforming it based on direction. - /// Forward: emit as-is. - /// Backward: swap begin/end types and flip handles. - void emit_event( - const DecompositionEvent& event, - bool forward, - const std::function& begin_chain, - const std::function& end_chain, - const std::function& begin_snarl, - const std::function& end_snarl - ) const; + /// Function that decides whether to flip a chain, given either of its + /// bounding node IDs. May be nondeterministic. + std::function should_flip; }; /** @@ -140,7 +146,7 @@ SnarlDecompositionFuzzer::SnarlDecompositionFuzzer( double p_flip, URNG& generator) : HandleGraphSnarlFinder(graph), wrapped(finder) { - should_flip = [&generator, p_flip](handle_t, handle_t) -> bool { + should_flip = [&generator, p_flip](nid_t ignored) -> bool { return std::uniform_real_distribution(0.0, 1.0)(generator) < p_flip; }; } From 836e7eed16b71e020ad9fc6b5c4cde17637cf924 Mon Sep 17 00:00:00 2001 From: Adam Novak Date: Fri, 13 Feb 2026 18:31:52 -0500 Subject: [PATCH 37/77] Hook up orientation fuzzers to random graph tests and fail to find more bugs --- src/unittest/snarl_distance_index.cpp | 60 ++++++++++++------- .../support/snarl_decomposition_fuzzer.cpp | 1 + 2 files changed, 41 insertions(+), 20 deletions(-) diff --git a/src/unittest/snarl_distance_index.cpp b/src/unittest/snarl_distance_index.cpp index 784a8f714a..3e7355c3f0 100644 --- a/src/unittest/snarl_distance_index.cpp +++ b/src/unittest/snarl_distance_index.cpp @@ -14,10 +14,13 @@ #include "catch.hpp" #include "support/random_graph.hpp" #include "support/randomness.hpp" +#include "support/randomly_flipped_nodes.hpp" +#include "support/snarl_decomposition_fuzzer.hpp" #include "../snarl_distance_index.hpp" #include "../integrated_snarl_finder.hpp" #include "../genotypekit.hpp" #include "../traversal_finder.hpp" +#include "../io/save_handle_graph.hpp" #include #include #include "xg.hpp" @@ -232,7 +235,7 @@ namespace vg { Edge* e17 = graph.create_edge(n11, n12); Edge* e18 = graph.create_edge(n12, n13); - graph.serialize_to_file("test_graph.vg"); + vg::io::save_handle_graph(&graph, "test_graph.vg"); //get the snarls IntegratedSnarlFinder snarl_finder(graph); SECTION("Traversal of chain") { @@ -7349,7 +7352,7 @@ namespace vg { << distance_index.minimum_distance(nodeID1, false, 0, node_id, true, 0) << " (" << dist_start_fd << " " << dist_end_fd << " " << dist_start_bk << " " << dist_end_bk << ") " << " is in the subgraph but shouldn't be " << endl; - graph.serialize_to_file("test_graph.vg"); + vg::io::save_handle_graph(&graph, "test_graph.vg"); } REQUIRE((start_forward || end_forward || in_forward || start_backward || end_backward || in_backward)); } else { @@ -7360,7 +7363,7 @@ namespace vg { << distance_index.minimum_distance(nodeID1, false, 0,node_id, true, 0) << " (" << dist_start_fd << " " << dist_end_fd << " " << dist_start_bk << " " << dist_end_bk << ") " << " is not in the subgraph but should be " << endl; - graph.serialize_to_file("test_graph.vg"); + vg::io::save_handle_graph(&graph, "test_graph.vg"); REQUIRE(!(start_forward || end_forward || in_forward || start_backward || end_backward || in_backward)); } } @@ -7429,27 +7432,45 @@ namespace vg { // Each actual graph takes a fairly long time to do so we randomize sizes... - default_random_engine generator(test_seed_source()); + std::default_random_engine generator(test_seed_source()); for (size_t repeat = 0; repeat < 1000; repeat++) { - uniform_int_distribution bases_dist(100, 1000); + std::uniform_int_distribution bases_dist(100, 1000); size_t bases = bases_dist(generator); - uniform_int_distribution variant_bases_dist(1, bases/20); + std::uniform_int_distribution variant_bases_dist(1, bases/20); size_t variant_bases = variant_bases_dist(generator); - uniform_int_distribution variant_count_dist(1, bases/30); + std::uniform_int_distribution variant_count_dist(1, bases/30); size_t variant_count = variant_count_dist(generator); + + std::uniform_real_distribution flip_dist(0.0, 1.0); + double node_flip_fraction = flip_dist(generator); + double chain_flip_fraction = flip_dist(generator); - uniform_int_distribution snarl_size_limit_dist(2, 1000); + std::uniform_int_distribution snarl_size_limit_dist(2, 1000); size_t size_limit = snarl_size_limit_dist(generator); - + #ifdef debug - cerr << repeat << ": Do graph of " << bases << " bp with ~" << variant_bases << " bp large variant length and " << variant_count << " events with size limit " << size_limit << endl; + cerr << repeat << ": Do graph of " << bases << " bp with ~" << variant_bases << " bp large variant length and " << variant_count << " events with " << node_flip_fraction << " nodes flipped and " << chain_flip_fraction << " of chains flipped, with size limit " << size_limit << endl; #endif - - VG graph; - random_graph(bases, variant_bases, variant_count, &graph); - IntegratedSnarlFinder finder(graph); + + // Generate a base graph + VG base_graph; + random_graph(bases, variant_bases, variant_count, &base_graph); + + // Flip some fraction of the nodes to their local reverse orientation + bdsg::HashGraph graph = randomly_flipped_nodes(base_graph, node_flip_fraction, generator); + + // Find snarls + IntegratedSnarlFinder base_finder(graph); + + // Flip some fraction of the chains to their opposite orientation. + // Note that we can't flip the snarls because the snarl decomposition + // requires snarls to be articulated as forward along their + // chains. + SnarlDecompositionFuzzer finder(&graph, &base_finder, chain_flip_fraction, generator); + + // Build the index SnarlDistanceIndex distance_index; fill_in_distance_index(&distance_index, &graph, &finder, size_limit); @@ -7509,7 +7530,7 @@ namespace vg { cerr << node_id1 << " " << (rev1 ? "rev" : "fd") << offset1 << " -> " << node_id2 << (rev2 ? "rev" : "fd") << offset2 << endl; cerr << "guessed: " << snarl_distance << " actual: " << dijkstra_distance << endl; cerr << "serializing graph to test_graph.vg" << endl; - graph.serialize_to_file("test_graph.vg"); + vg::io::save_handle_graph(&graph, "test_graph.vg"); REQUIRE(false); } if (max_distance < snarl_distance){ @@ -7517,11 +7538,10 @@ namespace vg { cerr << node_id1 << " " << (rev1 ? "rev" : "fd") << offset1 << " -> " << node_id2 << (rev2 ? "rev" : "fd") << offset2 << endl; cerr << "minimum: " << snarl_distance << " maximum: " << max_distance << endl; cerr << "serializing graph to test_graph.vg" << endl; - graph.serialize_to_file("test_graph.vg"); + vg::io::save_handle_graph(&graph, "test_graph.vg"); REQUIRE(false); } REQUIRE((snarl_distance >= dijkstra_distance || snarl_distance == std::numeric_limits::max())); - graph.serialize_to_file("test_graph.vg"); if (!traceback.first.empty() && ! traceback.second.empty()) { size_t traceback_distance = 0; for (auto x : traceback.first){ @@ -7568,7 +7588,7 @@ namespace vg { cerr << node_id1 << " " << (rev1 ? "rev" : "fd") << offset1 << " -> " << node_id2 << (rev2 ? "rev" : "fd") << offset2 << endl; cerr << "guessed: " << snarl_distance << " actual: " << dijkstra_distance << endl; cerr << "serializing graph to test_graph.vg" << endl; - graph.serialize_to_file("test_graph.vg"); + vg::io::save_handle_graph(&graph, "test_graph.vg"); REQUIRE(false); } REQUIRE((snarl_distance >= dijkstra_distance || snarl_distance == std::numeric_limits::max())); @@ -8012,7 +8032,7 @@ namespace vg { cerr << graph.get_id(start_handle) << (graph.get_is_reverse(start_handle) ? "rev" : "fd") << graph.get_length(start_handle) << " -> " << graph.get_id(end_handle) << (graph.get_is_reverse(end_handle) ? "rev" : "fd") << 0 << endl; cerr << "guessed: " << snarl_distance << " actual: " << dijkstra_distance << endl; cerr << "serializing graph to test_graph.vg" << endl; - vg::io::VPKG::save(graph, "test_graph.vg"); + vg::io::save_handle_graph(&graph, "test_graph.vg"); } REQUIRE(snarl_distance == dijkstra_distance); } @@ -8051,7 +8071,7 @@ namespace vg { SnarlDistanceIndex distance_index; fill_in_distance_index(&distance_index, &graph, &finder, size_limit); - graph.serialize_to_file("test_graph.vg"); + vg::io::save_handle_graph(&graph, "test_graph.vg"); for (size_t repeat_positions = 0 ; repeat_positions < 500 ; repeat_positions++) { //Pick random pairs of positions and find the distance between them id_t node_id1 = 0; diff --git a/src/unittest/support/snarl_decomposition_fuzzer.cpp b/src/unittest/support/snarl_decomposition_fuzzer.cpp index 4a3ca3583c..6b49a5b660 100644 --- a/src/unittest/support/snarl_decomposition_fuzzer.cpp +++ b/src/unittest/support/snarl_decomposition_fuzzer.cpp @@ -39,6 +39,7 @@ void SnarlDecompositionFuzzer::traverse_decomposition( emit_event(flip(event, graph), false); } else { // Call the right handler on the event's handle. + // TODO: Is this really better than a nice clear switch??? (*handlers.at((int)event.type))(event.handle); } }; From 22ff0c440c3906acdc972ec66543491b86c6870b Mon Sep 17 00:00:00 2001 From: Adam Novak Date: Fri, 13 Feb 2026 19:05:11 -0500 Subject: [PATCH 38/77] Dump mostly-synthetic hot tips for cool robots --- BOTS.md | 75 +++++++++++++++++++++++++++++++++++++++++++++++++++++++ CLAUDE.md | 1 + 2 files changed, 76 insertions(+) create mode 100644 BOTS.md create mode 120000 CLAUDE.md diff --git a/BOTS.md b/BOTS.md new file mode 100644 index 0000000000..2c35275ac0 --- /dev/null +++ b/BOTS.md @@ -0,0 +1,75 @@ +# VG Project Notes + +## Building +- New `.cpp` files auto-discovered +- Build with `make -j8` or `make obj/whatever.o` to build just one .o. +- You may be getting errors from `clangd`. If these errors seem spurious, stop and demand a `clangd` that works properly. + +## Testing + +### Running Bash-TAP Tests +Use `prove -v` (not `bash`) to execute Bash-TAP tests. This provides proper test harness output and better error reporting. + +**Important**: Run `prove` from the `test/` directory: +```bash +cd test +prove -v t/26_deconstruct.t +``` + +### Running Unit Tests +To run all unit tests: +```bash +./bin/vg test +``` +- `./bin/vg test "[tag]"` runs tests matching a tag + +#### Writing Unit Tests +- Framework: Catch v2 (header-only) +- Include: `#include "catch.hpp"` (in `src/unittest/catch.hpp`) +- Macros: `TEST_CASE("name", "[tags]")`, `SECTION("name")`, `REQUIRE(cond)` +- Namespace: `vg::unittest` +- Directory: `src/unittest/` + +### Running All Tests +```bash +make test +``` + +## Writing Code + +### HandleGraph API +The interfaces in libhandlegraph model a bidirected sequence graph (where nodes have DNA sequences and edges can connect to either the start or end of each involved node). + +#### Core types +- `handle_t` - opaque 64-bit value +- `nid_t` - node ID type +- `edge_t` = `pair` + +#### Key HandleGraph methods +- `get_handle(nid_t, bool is_reverse=false)` → `handle_t` +- `get_id(handle_t)` → `nid_t` +- `get_is_reverse(handle_t)` → `bool` +- `flip(handle_t)` → `handle_t` (toggle orientation) +- `get_sequence(handle_t)` → `string` (in handle's orientation) +- `follow_edges(handle_t, bool go_left, iteratee)` - iterate neighbors +- `for_each_handle(iteratee, bool parallel=false)` - iterate all nodes +- `for_each_edge(iteratee, bool parallel=false)` - iterate all edges +- `has_edge(handle_t left, handle_t right)` → `bool` + +#### MutableHandleGraph additions +- `create_handle(string seq)` / `create_handle(string seq, nid_t id)` → `handle_t` +- `create_edge(handle_t left, handle_t right)` +- `destroy_handle(handle_t)` / `destroy_edge(handle_t, handle_t)` + +#### HandleGraph algorithms +- Things like `topological_sort.hpp` and copy_graph.hpp` are in `deps/libhandlegraph/src/include/handlegraph/algorithms`. + +#### bdsg::HashGraph +- Header: `deps/libbdsg/bdsg/include/bdsg/hash_graph.hpp` +- Implements MutablePathMutableHandleGraph +- Go-to handlegraph implementation to use +- In libbdsg + +### Utilities +- `reverse_complement(string)` → `string` in src/utility.hpp + diff --git a/CLAUDE.md b/CLAUDE.md new file mode 120000 index 0000000000..1a1007d91a --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1 @@ +BOTS.md \ No newline at end of file From e92a036c52bacf84faebcc28bf98e71de0a1621c Mon Sep 17 00:00:00 2001 From: Adam Novak Date: Fri, 20 Mar 2026 12:41:01 -0700 Subject: [PATCH 39/77] Implement populating is_regular and the single strict notion of regularity --- deps/libbdsg | 2 +- src/snarl_distance_index.cpp | 150 +++++++++++++++++++++++--- src/unittest/snarl_distance_index.cpp | 51 +++++++-- src/zip_code.cpp | 2 +- 4 files changed, 178 insertions(+), 27 deletions(-) diff --git a/deps/libbdsg b/deps/libbdsg index f522ff9a20..1522fa93e5 160000 --- a/deps/libbdsg +++ b/deps/libbdsg @@ -1 +1 @@ -Subproject commit f522ff9a20654627b9a803ba4b0fd33cc4f54dbc +Subproject commit 1522fa93e502fdd5f301fd7b4b048e49c52d53fd diff --git a/src/snarl_distance_index.cpp b/src/snarl_distance_index.cpp index 16193b931e..d8240145f3 100644 --- a/src/snarl_distance_index.cpp +++ b/src/snarl_distance_index.cpp @@ -371,12 +371,12 @@ SnarlDistanceIndex::TemporaryDistanceIndex make_temporary_distance_index( temp_snarl_record.end_node_length = graph->get_length(snarl_end_handle); temp_snarl_record.node_count = temp_snarl_record.children.size(); bool any_edges_in_snarl = false; - graph->follow_edges(graph->get_handle(temp_snarl_record.start_node_id, temp_snarl_record.start_node_rev), false, [&](const handle_t next_handle) { + graph->follow_edges(graph->get_handle(temp_snarl_record.start_node_id, temp_snarl_record.start_node_rev), false, [&](const handle_t& next_handle) { if (graph->get_id(next_handle) != temp_snarl_record.end_node_id) { any_edges_in_snarl = true; } }); - graph->follow_edges(graph->get_handle(temp_snarl_record.end_node_id, !temp_snarl_record.end_node_rev), false, [&](const handle_t next_handle) { + graph->follow_edges(graph->get_handle(temp_snarl_record.end_node_id, !temp_snarl_record.end_node_rev), false, [&](const handle_t& next_handle) { if (graph->get_id(next_handle) != temp_snarl_record.start_node_id) { any_edges_in_snarl = true; } @@ -578,7 +578,7 @@ SnarlDistanceIndex::TemporaryDistanceIndex make_temporary_distance_index( //Snarls get counted as trivial if they contain no nodes but they might still have edges size_t backward_loop = std::numeric_limits::max(); - graph->follow_edges(graph->get_handle(temp_node_record.node_id, !temp_node_record.reversed_in_parent), false, [&](const handle_t next_handle) { + graph->follow_edges(graph->get_handle(temp_node_record.node_id, !temp_node_record.reversed_in_parent), false, [&](const handle_t& next_handle) { if (graph->get_id(next_handle) == temp_node_record.node_id) { //If there is a loop going backwards (relative to the chain) back to the same node backward_loop = 0; @@ -668,7 +668,7 @@ SnarlDistanceIndex::TemporaryDistanceIndex make_temporary_distance_index( //Check if there is a loop in this node //Snarls get counted as trivial if they contain no nodes but they might still have edges size_t forward_loop = std::numeric_limits::max(); - graph->follow_edges(graph->get_handle(temp_node_record.node_id, temp_node_record.reversed_in_parent), false, [&](const handle_t next_handle) { + graph->follow_edges(graph->get_handle(temp_node_record.node_id, temp_node_record.reversed_in_parent), false, [&](const handle_t& next_handle) { if (graph->get_id(next_handle) == temp_node_record.node_id) { //If there is a loop going forward (relative to the chain) back to the same node forward_loop = 0; @@ -811,6 +811,16 @@ static void populate_distance_matrix_if_needed(SnarlDistanceIndex::TemporaryDist */ static void populate_hub_labeling(SnarlDistanceIndex::TemporaryDistanceIndex& temp_index, SnarlDistanceIndex::temp_record_ref_t& snarl_index, SnarlDistanceIndex::TemporaryDistanceIndex::TemporarySnarlRecord& temp_snarl_record, vector& all_children, const HandleGraph* graph); +/** + * Determine if a snarl is regular or not. + * + * A regular snarl is a snarl that, while not simple, consists of only nodes or + * chains connected to the start and end, without any connections between + * multiple children, or any way to turn around. THere may be an edge directly + * across. + */ +static bool check_regularity(const SnarlDistanceIndex::TemporaryDistanceIndex& temp_index, const SnarlDistanceIndex::temp_record_ref_t& snarl_index, const SnarlDistanceIndex::TemporaryDistanceIndex::TemporarySnarlRecord& temp_snarl_record, const vector& all_children, const HandleGraph* graph); + /** * Fill in the snarl index. * The index will already know its boundaries and everything knows their relationships in the @@ -829,9 +839,6 @@ void populate_snarl_index( SnarlDistanceIndex::TemporaryDistanceIndex::TemporarySnarlRecord& temp_snarl_record = temp_index.get_snarl(snarl_index); temp_snarl_record.is_simple=true; - - - /*Helper function to find the ancestor of a node that is a child of this snarl */ auto get_ancestor_of_node = [&](SnarlDistanceIndex::temp_record_ref_t curr_index, SnarlDistanceIndex::temp_record_ref_t ancestor_snarl_index) { @@ -927,7 +934,7 @@ void populate_snarl_index( } //Add everything reachable from the start boundary node that has no other incoming edges - graph->follow_edges(current_graph_handle, false, [&](const handle_t next_handle) { + graph->follow_edges(current_graph_handle, false, [&](const handle_t& next_handle) { #ifdef debug_distance_indexing cerr << "Following forward edges from " << graph->get_id(current_graph_handle) << " to " << graph->get_id(next_handle) << endl; #endif @@ -967,7 +974,7 @@ void populate_snarl_index( //Does this have no unseen incoming edges but including nodes we've seen in the other direction? //TODO: Actually do this - graph->follow_edges(reverse_handle, false, [&](const handle_t incoming_handle) { + graph->follow_edges(reverse_handle, false, [&](const handle_t& incoming_handle) { #ifdef debug_distance_indexing cerr << "Getting backwards edge to " << graph->get_id(incoming_handle) << endl; #endif @@ -1085,9 +1092,9 @@ void populate_snarl_index( populate_distance_matrix_if_needed(temp_index, snarl_index, temp_snarl_record, all_children, graph, size_limit, only_top_level_chain_distances); } - //If this is a simple snarl (one with only single nodes that connect to the start and end nodes), then - // we want to remember if the child nodes are reversed if (temp_snarl_record.is_simple) { + // If this is a simple snarl (one with only single nodes that connect to the start and end nodes), then + // we want to remember if the child nodes are reversed for (size_t i = 0 ; i < temp_snarl_record.node_count ; i++) { //Get the index of the child const SnarlDistanceIndex::temp_record_ref_t& child_index = temp_snarl_record.children[i]; @@ -1105,9 +1112,12 @@ void populate_snarl_index( //Set the orientation of this node in the simple snarl temp_node_record.reversed_in_parent = temp_node_record.distance_left_start == std::numeric_limits::max(); - } - } + + } + + // Decide if the snarl is regular. + temp_snarl_record.is_regular = check_regularity(temp_index, snarl_index, temp_snarl_record, all_children, graph); //Now that the distances are filled in, predict the size of the snarl in the index temp_index.max_index_size += temp_snarl_record.get_max_record_length(); @@ -1212,7 +1222,7 @@ void populate_distance_matrix_if_needed(SnarlDistanceIndex::TemporaryDistanceInd : temp_index.get_chain(start_index).rank_in_parent; bool has_edges = false; - graph->follow_edges(graph->get_handle(node_id, false), false, [&](const handle_t next_handle) { + graph->follow_edges(graph->get_handle(node_id, false), false, [&](const handle_t& next_handle) { has_edges = true; }); if (!has_edges) { @@ -1221,7 +1231,7 @@ void populate_distance_matrix_if_needed(SnarlDistanceIndex::TemporaryDistanceInd temp_snarl_record.is_simple=false; //It is a tip so this isn't simple snarl } has_edges = false; - graph->follow_edges(graph->get_handle(node_id, true), false, [&](const handle_t next_handle) { + graph->follow_edges(graph->get_handle(node_id, true), false, [&](const handle_t& next_handle) { has_edges = true; }); if (!has_edges) { @@ -1358,7 +1368,7 @@ void populate_distance_matrix_row(SnarlDistanceIndex::TemporaryDistanceIndex& te << (current_rev ? "rev" : "fd") << " at actual node " << graph->get_id(current_end_handle) << (graph->get_is_reverse(current_end_handle) ? "rev" : "fd") << endl; #endif - graph->follow_edges(current_end_handle, false, [&](const handle_t next_handle) { + graph->follow_edges(current_end_handle, false, [&](const handle_t& next_handle) { if (graph->get_id(current_end_handle) == graph->get_id(next_handle)){ //If this loops onto the same node side then this isn't a simple snarl temp_snarl_record.is_simple = false; @@ -1602,6 +1612,114 @@ void populate_distance_matrix_row(SnarlDistanceIndex::TemporaryDistanceIndex& te } } +bool check_regularity(const SnarlDistanceIndex::TemporaryDistanceIndex& temp_index, const SnarlDistanceIndex::temp_record_ref_t& snarl_index, const SnarlDistanceIndex::TemporaryDistanceIndex::TemporarySnarlRecord& temp_snarl_record, const vector& all_children, const HandleGraph* graph) { + if (temp_snarl_record.is_root_snarl) { + // Roots can't be regular. + return false; + } + if (temp_snarl_record.is_simple) { + // Simple snarls can't be regular because simple is more specific and useful. + return false; + } + + // Get the snarl boundary nodes, facing out + handle_t start_out = graph->get_handle(temp_snarl_record.start_node_id, !temp_snarl_record.start_node_rev); + handle_t end_out = graph->get_handle(temp_snarl_record.end_node_id, temp_snarl_record.end_node_rev); + + // Define accessors to get bounding graph handles for children, facing out. + auto child_start_out = [&](const SnarlDistanceIndex::temp_record_ref_t& child_index) { + return child_index.first == SnarlDistanceIndex::TEMP_NODE ? + graph->get_handle(child_index.second, true) : + graph->get_handle( + temp_index.get_chain(child_index).start_node_id, + !temp_index.get_chain(child_index).start_node_rev + ); + }; + auto child_end_out = [&](const SnarlDistanceIndex::temp_record_ref_t& child_index) { + return child_index.first == SnarlDistanceIndex::TEMP_NODE ? + graph->get_handle(child_index.second, false) : + graph->get_handle( + temp_index.get_chain(child_index).end_node_id, + temp_index.get_chain(child_index).end_node_rev + ); + }; + + for (const SnarlDistanceIndex::temp_record_ref_t& child_index : all_children) { + // Have we seen the snarl start? + bool saw_start = false; + // Have we seen the snarl end? + bool saw_end = false; + // Have we seen anything else, or a duplicate snarl boundary? + bool saw_other = false; + + auto handle_destination = [&](const handle_t& next_handle) { + // Every edge out the end the child must go to a snarl boundary out + // that hasn't been reached yet. + if (next_handle == start_out && !saw_start) { + saw_start = true; + return true; + } else if (next_handle == end_out && !saw_end) { + saw_end = true; + return true; + } else { + saw_other = true; + // We don't care if we have an edge going the right way because + // we found an edge going the wrong way. + return false; + } + }; + + // Check the edges off the child start + graph->follow_edges(child_start_out(child_index), false, handle_destination); + + if (saw_other || !(saw_start != saw_end)) { + // We have an edge we shouldn't, or we don't connect to exactly one boundary. + return false; + } + + // Check the edges off the child end + graph->follow_edges(child_end_out(child_index), false, handle_destination); + + if (saw_other || !saw_start || !saw_end) { + // We have an edge we shouldn't, or we haven't reached both + // boundaries exactly once across the two ends of the child. + return false; + } + + if (child_index.first == SnarlDistanceIndex::TEMP_CHAIN) { + // If a child is a chain, check it for loops + const SnarlDistanceIndex::TemporaryDistanceIndex::TemporaryChainRecord& temp_chain_record = temp_index.get_chain(child_index); + if (!temp_chain_record.forward_loops.empty() && temp_chain_record.forward_loops.front() != std::numeric_limits::max()) { + // There's a forward loop in this child chain, so the snarl's not regular. + return false; + } + if (!temp_chain_record.backward_loops.empty() && temp_chain_record.backward_loops.back() != std::numeric_limits::max()) { + // There's a backward loop in this child chain, so the snarl's not regular. + return false; + } + } + } + + // Now we know the children are fine; check for disallowed edges between + // the sentinels. + + handle_t start_in = graph->flip(start_out); + if (graph->has_edge(start_in, start_out)) { + return false; + } + + handle_t end_in = graph->flip(end_out); + if (graph->has_edge(end_in, end_out)) { + return false; + } + + // If we don't have any disallowed edges, and we don't have any children + // without the exact right connectivity, we must be regular. + + // We don't make sure we actually had any children. + return true; +} + //Given an alignment to a graph and a range, find the set of nodes in the diff --git a/src/unittest/snarl_distance_index.cpp b/src/unittest/snarl_distance_index.cpp index 3e7355c3f0..5334fba7c3 100644 --- a/src/unittest/snarl_distance_index.cpp +++ b/src/unittest/snarl_distance_index.cpp @@ -197,7 +197,42 @@ namespace vg { REQUIRE(distance_index.minimum_distance(2, true, 0, 2, true, 1) == 1); } } - TEST_CASE( "Nested chain with loop", "[snarl_distance]" ) { + TEST_CASE( "Can distance index nested chain without loop", "[snarl_distance]" ) { + bdsg::HashGraph graph; + handle_t h1 = graph.create_handle("G"); + handle_t h2 = graph.create_handle("A"); + handle_t h3 = graph.create_handle("T"); + handle_t h4 = graph.create_handle("T"); + handle_t h5 = graph.create_handle("A"); + handle_t h6 = graph.create_handle("C"); + handle_t h7 = graph.create_handle("A"); + + // Wire it up as a stick + graph.create_edge(h1, h2); + graph.create_edge(h2, h3); + graph.create_edge(h3, h4); + graph.create_edge(h4, h5); + graph.create_edge(h5, h6); + graph.create_edge(h6, h7); + + // Allow skipping a run of nodes to make a snarl with a child chain + graph.create_edge(h2, h5); + + IntegratedSnarlFinder snarl_finder(graph); + + SECTION("Snarl classifications are correct") { + SECTION("Distance index") { + SnarlDistanceIndex distance_index; + fill_in_distance_index(&distance_index, &graph, &snarl_finder); + REQUIRE(distance_index.is_regular_snarl(distance_index.get_parent(distance_index.get_parent(distance_index.get_node_net_handle(graph.get_id(h3)))))); + } SECTION("Distanceless index") { + SnarlDistanceIndex distance_index; + fill_in_distance_index(&distance_index, &graph, &snarl_finder, 0); + REQUIRE(distance_index.is_regular_snarl(distance_index.get_parent(distance_index.get_parent(distance_index.get_node_net_handle(graph.get_id(h3)))))); + } + } + } + TEST_CASE( "Can distance index nested chain with loop", "[snarl_distance]" ) { VG graph; @@ -235,7 +270,8 @@ namespace vg { Edge* e17 = graph.create_edge(n11, n12); Edge* e18 = graph.create_edge(n12, n13); - vg::io::save_handle_graph(&graph, "test_graph.vg"); + //vg::io::save_handle_graph(&graph, "test_graph.vg"); + //get the snarls IntegratedSnarlFinder snarl_finder(graph); SECTION("Traversal of chain") { @@ -253,16 +289,13 @@ namespace vg { fill_in_distance_index(&distance_index, &graph, &snarl_finder); REQUIRE(!distance_index.is_regular_snarl(distance_index.get_parent(distance_index.get_parent(distance_index.get_node_net_handle(n3->id()))))); REQUIRE(!distance_index.is_regular_snarl(distance_index.get_parent(distance_index.get_parent(distance_index.get_node_net_handle(n8->id()))))); - REQUIRE(distance_index.is_regular_snarl(distance_index.get_parent(distance_index.get_parent(distance_index.get_node_net_handle(n6->id()))), true)); - REQUIRE(!distance_index.is_regular_snarl(distance_index.get_parent(distance_index.get_parent(distance_index.get_node_net_handle(n6->id()))), false)); + REQUIRE(!distance_index.is_regular_snarl(distance_index.get_parent(distance_index.get_parent(distance_index.get_node_net_handle(n6->id()))))); } SECTION("Distanceless index") { SnarlDistanceIndex distance_index; fill_in_distance_index(&distance_index, &graph, &snarl_finder, 0); - REQUIRE(!distance_index.is_regular_snarl(distance_index.get_parent(distance_index.get_parent(distance_index.get_node_net_handle(n3->id()))), true, &graph)); - REQUIRE(!distance_index.is_regular_snarl(distance_index.get_parent(distance_index.get_parent(distance_index.get_node_net_handle(n8->id()))), true, &graph)); - REQUIRE(distance_index.is_regular_snarl(distance_index.get_parent(distance_index.get_parent(distance_index.get_node_net_handle(n6->id()))), true, &graph)); - // TODO: This isn't true because it would be too much work to recursively check all children using only the graph - //REQUIRE(!distance_index.is_regular_snarl(distance_index.get_parent(distance_index.get_parent(distance_index.get_node_net_handle(n6->id()))), false, &graph)); + REQUIRE(!distance_index.is_regular_snarl(distance_index.get_parent(distance_index.get_parent(distance_index.get_node_net_handle(n3->id()))))); + REQUIRE(!distance_index.is_regular_snarl(distance_index.get_parent(distance_index.get_parent(distance_index.get_node_net_handle(n8->id()))))); + REQUIRE(!distance_index.is_regular_snarl(distance_index.get_parent(distance_index.get_parent(distance_index.get_node_net_handle(n6->id()))))); } } SECTION("Minimum distances are correct") { diff --git a/src/zip_code.cpp b/src/zip_code.cpp index 4699a24494..14b97b102a 100644 --- a/src/zip_code.cpp +++ b/src/zip_code.cpp @@ -121,7 +121,7 @@ void ZipCode::fill_in_zipcode_from_pos(const SnarlDistanceIndex& distance_index, } return; } - } else if (distance_index.is_regular_snarl(current_ancestor, false, graph_ptr)) { + } else if (distance_index.is_regular_snarl(current_ancestor)) { snarl_code_t snarl_code = get_regular_snarl_code(current_ancestor, ancestors[i-1], distance_index); zipcode.add_value(snarl_code.get_raw_code_type()); zipcode.add_value(snarl_code.get_raw_prefix_sum_or_identifier()); From e4e1dae913ac0775a9ec5dedb32e45e312dffcc4 Mon Sep 17 00:00:00 2001 From: Adam Novak Date: Fri, 20 Mar 2026 13:59:19 -0700 Subject: [PATCH 40/77] Add debugging and reduce Saturn levels by not consuming all_children --- deps/libbdsg | 2 +- src/snarl_distance_index.cpp | 82 ++++++++++++++++++++++----- src/unittest/snarl_distance_index.cpp | 2 +- 3 files changed, 70 insertions(+), 16 deletions(-) diff --git a/deps/libbdsg b/deps/libbdsg index 1522fa93e5..46aa0941b7 160000 --- a/deps/libbdsg +++ b/deps/libbdsg @@ -1 +1 @@ -Subproject commit 1522fa93e502fdd5f301fd7b4b048e49c52d53fd +Subproject commit 46aa0941b73b809879c7c0a6a7f239cbbe6d45b8 diff --git a/src/snarl_distance_index.cpp b/src/snarl_distance_index.cpp index d8240145f3..0a1517b3e2 100644 --- a/src/snarl_distance_index.cpp +++ b/src/snarl_distance_index.cpp @@ -1,4 +1,4 @@ -//#define debug_distance_indexing +#define debug_distance_indexing //#define debug_snarl_traversal //#define debug_distances //#define debug_subgraph @@ -792,7 +792,7 @@ SnarlDistanceIndex::TemporaryDistanceIndex make_temporary_distance_index( * Populate a row of the distance matrix. * Also responsible for filling in min_length, distance_start_start, and distance_start_end on the TemporarySnarlRecord when a distance matrix is used. */ -static void populate_distance_matrix_row(SnarlDistanceIndex::TemporaryDistanceIndex& temp_index, SnarlDistanceIndex::temp_record_ref_t& snarl_index, SnarlDistanceIndex::TemporaryDistanceIndex::TemporarySnarlRecord& temp_snarl_record, const SnarlDistanceIndex::temp_record_ref_t& start_index, const HandleGraph* graph, size_t start_rank, bool is_internal_node, size_t size_limit); +static void populate_distance_matrix_row(SnarlDistanceIndex::TemporaryDistanceIndex& temp_index, const SnarlDistanceIndex::temp_record_ref_t& snarl_index, SnarlDistanceIndex::TemporaryDistanceIndex::TemporarySnarlRecord& temp_snarl_record, const SnarlDistanceIndex::temp_record_ref_t& start_index, const HandleGraph* graph, size_t start_rank, bool is_internal_node, size_t size_limit); /** * Fills in required distance matrix rows for each child @@ -801,7 +801,7 @@ static void populate_distance_matrix_row(SnarlDistanceIndex::TemporaryDistanceIn * - size_limit == 0: no distances in index, so no rows * - Top-level chain distances only: ??? */ -static void populate_distance_matrix_if_needed(SnarlDistanceIndex::TemporaryDistanceIndex& temp_index, SnarlDistanceIndex::temp_record_ref_t& snarl_index, SnarlDistanceIndex::TemporaryDistanceIndex::TemporarySnarlRecord& temp_snarl_record, vector& all_children, const HandleGraph* graph, size_t size_limit, bool only_top_level_chain_distances); +static void populate_distance_matrix_if_needed(SnarlDistanceIndex::TemporaryDistanceIndex& temp_index, const SnarlDistanceIndex::temp_record_ref_t& snarl_index, SnarlDistanceIndex::TemporaryDistanceIndex::TemporarySnarlRecord& temp_snarl_record, const vector& all_children, const HandleGraph* graph, size_t size_limit, bool only_top_level_chain_distances); /** * Does three things: @@ -809,14 +809,14 @@ static void populate_distance_matrix_if_needed(SnarlDistanceIndex::TemporaryDist * - Builds the hub labels * - Stores labels in temp_snarl_record */ -static void populate_hub_labeling(SnarlDistanceIndex::TemporaryDistanceIndex& temp_index, SnarlDistanceIndex::temp_record_ref_t& snarl_index, SnarlDistanceIndex::TemporaryDistanceIndex::TemporarySnarlRecord& temp_snarl_record, vector& all_children, const HandleGraph* graph); +static void populate_hub_labeling(SnarlDistanceIndex::TemporaryDistanceIndex& temp_index, const SnarlDistanceIndex::temp_record_ref_t& snarl_index, SnarlDistanceIndex::TemporaryDistanceIndex::TemporarySnarlRecord& temp_snarl_record, const vector& all_children, const HandleGraph* graph); /** * Determine if a snarl is regular or not. * * A regular snarl is a snarl that, while not simple, consists of only nodes or * chains connected to the start and end, without any connections between - * multiple children, or any way to turn around. THere may be an edge directly + * multiple children, or any way to turn around. There may be an edge directly * across. */ static bool check_regularity(const SnarlDistanceIndex::TemporaryDistanceIndex& temp_index, const SnarlDistanceIndex::temp_record_ref_t& snarl_index, const SnarlDistanceIndex::TemporaryDistanceIndex::TemporarySnarlRecord& temp_snarl_record, const vector& all_children, const HandleGraph* graph); @@ -1129,7 +1129,7 @@ void populate_snarl_index( temp_index.max_bits = std::max(temp_index.max_bits, 22 + SnarlDistanceIndex::bit_width(temp_snarl_record.children.size())); } -void populate_hub_labeling(SnarlDistanceIndex::TemporaryDistanceIndex& temp_index, SnarlDistanceIndex::temp_record_ref_t& snarl_index, SnarlDistanceIndex::TemporaryDistanceIndex::TemporarySnarlRecord& temp_snarl_record, vector& all_children, const HandleGraph* graph) { +void populate_hub_labeling(SnarlDistanceIndex::TemporaryDistanceIndex& temp_index, const SnarlDistanceIndex::temp_record_ref_t& snarl_index, SnarlDistanceIndex::TemporaryDistanceIndex::TemporarySnarlRecord& temp_snarl_record, const vector& all_children, const HandleGraph* graph) { CHOverlay ov = make_boost_graph(temp_index, snarl_index, temp_snarl_record, all_children, graph); #ifdef debug_hub_label_build @@ -1192,7 +1192,7 @@ void populate_hub_labeling(SnarlDistanceIndex::TemporaryDistanceIndex& temp_inde #endif } -void populate_distance_matrix_if_needed(SnarlDistanceIndex::TemporaryDistanceIndex& temp_index, SnarlDistanceIndex::temp_record_ref_t& snarl_index, SnarlDistanceIndex::TemporaryDistanceIndex::TemporarySnarlRecord& temp_snarl_record, vector& all_children, const HandleGraph* graph, size_t size_limit, bool only_top_level_chain_distances) { +void populate_distance_matrix_if_needed(SnarlDistanceIndex::TemporaryDistanceIndex& temp_index, const SnarlDistanceIndex::temp_record_ref_t& snarl_index, SnarlDistanceIndex::TemporaryDistanceIndex::TemporarySnarlRecord& temp_snarl_record, const vector& all_children, const HandleGraph* graph, size_t size_limit, bool only_top_level_chain_distances) { if (size_limit != 0 && !only_top_level_chain_distances) { //If we are saving distances //Reserve enough space to store all possible distances @@ -1201,10 +1201,10 @@ void populate_distance_matrix_if_needed(SnarlDistanceIndex::TemporaryDistanceInd : temp_snarl_record.node_count * temp_snarl_record.node_count); } else { temp_snarl_record.include_distances = false; - } - while (!all_children.empty()) { - const SnarlDistanceIndex::temp_record_ref_t start_index = std::move(all_children.back()); - all_children.pop_back(); + } + for (auto it = all_children.rbegin(); it != all_children.rend(); ++it) { + // Visit all the children in reverse order + const SnarlDistanceIndex::temp_record_ref_t& start_index = *it; bool is_internal_node = false; @@ -1278,7 +1278,7 @@ void populate_distance_matrix_if_needed(SnarlDistanceIndex::TemporaryDistanceInd -void populate_distance_matrix_row(SnarlDistanceIndex::TemporaryDistanceIndex& temp_index, SnarlDistanceIndex::temp_record_ref_t& snarl_index, SnarlDistanceIndex::TemporaryDistanceIndex::TemporarySnarlRecord& temp_snarl_record, const SnarlDistanceIndex::temp_record_ref_t& start_index, const HandleGraph* graph, size_t start_rank, bool is_internal_node, size_t size_limit) { +void populate_distance_matrix_row(SnarlDistanceIndex::TemporaryDistanceIndex& temp_index, const SnarlDistanceIndex::temp_record_ref_t& snarl_index, SnarlDistanceIndex::TemporaryDistanceIndex::TemporarySnarlRecord& temp_snarl_record, const SnarlDistanceIndex::temp_record_ref_t& start_index, const HandleGraph* graph, size_t start_rank, bool is_internal_node, size_t size_limit) { /*Helper function to find the ancestor of a node that is a child of this snarl */ auto get_ancestor_of_node = [&](SnarlDistanceIndex::temp_record_ref_t curr_index, SnarlDistanceIndex::temp_record_ref_t ancestor_snarl_index) { @@ -1613,12 +1613,22 @@ void populate_distance_matrix_row(SnarlDistanceIndex::TemporaryDistanceIndex& te } bool check_regularity(const SnarlDistanceIndex::TemporaryDistanceIndex& temp_index, const SnarlDistanceIndex::temp_record_ref_t& snarl_index, const SnarlDistanceIndex::TemporaryDistanceIndex::TemporarySnarlRecord& temp_snarl_record, const vector& all_children, const HandleGraph* graph) { +#ifdef debug_distance_indexing + std::cerr << "Check if snarl " << temp_snarl_record.start_node_id << " to " << temp_snarl_record.end_node_id << " with " << all_children.size() << " children is regular" << std::endl; +#endif + if (temp_snarl_record.is_root_snarl) { // Roots can't be regular. +#ifdef debug_distance_indexing + std::cerr << "Snarl is not regular because it is a root snarl." << std::endl; +#endif return false; } if (temp_snarl_record.is_simple) { // Simple snarls can't be regular because simple is more specific and useful. +#ifdef debug_distance_indexing + std::cerr << "Snarl is not regular because it is simple." << std::endl; +#endif return false; } @@ -1653,36 +1663,63 @@ bool check_regularity(const SnarlDistanceIndex::TemporaryDistanceIndex& temp_ind bool saw_other = false; auto handle_destination = [&](const handle_t& next_handle) { +#ifdef debug_distance_indexing + std::cerr << "\tConnects to " << graph->get_id(next_handle) << (graph->get_is_reverse(next_handle) ? "-" : "+") << std::endl; +#endif + // Every edge out the end the child must go to a snarl boundary out // that hasn't been reached yet. if (next_handle == start_out && !saw_start) { saw_start = true; +#ifdef debug_distance_indexing + std::cerr << "\t\tThis is a new connection to snarl start" << std::endl; +#endif return true; } else if (next_handle == end_out && !saw_end) { saw_end = true; +#ifdef debug_distance_indexing + std::cerr << "\t\tThis is a new connection to snarl end" << std::endl; +#endif return true; } else { saw_other = true; // We don't care if we have an edge going the right way because // we found an edge going the wrong way. +#ifdef debug_distance_indexing + std::cerr << "\t\tThis is an unwanted connection!" << std::endl; +#endif return false; } }; // Check the edges off the child start - graph->follow_edges(child_start_out(child_index), false, handle_destination); + handle_t here = child_start_out(child_index); +#ifdef debug_distance_indexing + std::cerr << "Look right from " << graph->get_id(here) << (graph->get_is_reverse(here) ? "-" : "+") << std::endl; +#endif + graph->follow_edges(here, false, handle_destination); if (saw_other || !(saw_start != saw_end)) { // We have an edge we shouldn't, or we don't connect to exactly one boundary. +#ifdef debug_distance_indexing + std::cerr << "\tWe must not be regular" << std::endl; +#endif return false; } // Check the edges off the child end - graph->follow_edges(child_end_out(child_index), false, handle_destination); + here = child_end_out(child_index); +#ifdef debug_distance_indexing + std::cerr << "Look right from " << graph->get_id(here) << (graph->get_is_reverse(here) ? "-" : "+") << std::endl; +#endif + graph->follow_edges(here, false, handle_destination); if (saw_other || !saw_start || !saw_end) { // We have an edge we shouldn't, or we haven't reached both // boundaries exactly once across the two ends of the child. +#ifdef debug_distance_indexing + std::cerr << "\tWe must not be regular" << std::endl; +#endif return false; } @@ -1691,10 +1728,16 @@ bool check_regularity(const SnarlDistanceIndex::TemporaryDistanceIndex& temp_ind const SnarlDistanceIndex::TemporaryDistanceIndex::TemporaryChainRecord& temp_chain_record = temp_index.get_chain(child_index); if (!temp_chain_record.forward_loops.empty() && temp_chain_record.forward_loops.front() != std::numeric_limits::max()) { // There's a forward loop in this child chain, so the snarl's not regular. +#ifdef debug_distance_indexing + std::cerr << "We are not regular because there's a forward loop in this child chain." << std::endl; +#endif return false; } if (!temp_chain_record.backward_loops.empty() && temp_chain_record.backward_loops.back() != std::numeric_limits::max()) { // There's a backward loop in this child chain, so the snarl's not regular. +#ifdef debug_distance_indexing + std::cerr << "We are not regular because there's a backward loop in this child chain." << std::endl; +#endif return false; } } @@ -1705,11 +1748,17 @@ bool check_regularity(const SnarlDistanceIndex::TemporaryDistanceIndex& temp_ind handle_t start_in = graph->flip(start_out); if (graph->has_edge(start_in, start_out)) { +#ifdef debug_distance_indexing + std::cerr << "We are not regular because we have a start-start loop." << std::endl; +#endif return false; } handle_t end_in = graph->flip(end_out); if (graph->has_edge(end_in, end_out)) { +#ifdef debug_distance_indexing + std::cerr << "We are not regular because we have an end-end loop." << std::endl; +#endif return false; } @@ -1717,6 +1766,11 @@ bool check_regularity(const SnarlDistanceIndex::TemporaryDistanceIndex& temp_ind // without the exact right connectivity, we must be regular. // We don't make sure we actually had any children. + +#ifdef debug_distance_indexing + std::cerr << "We are a regular snarl." << std::endl; +#endif + return true; } diff --git a/src/unittest/snarl_distance_index.cpp b/src/unittest/snarl_distance_index.cpp index 5334fba7c3..f540c7a02b 100644 --- a/src/unittest/snarl_distance_index.cpp +++ b/src/unittest/snarl_distance_index.cpp @@ -270,7 +270,7 @@ namespace vg { Edge* e17 = graph.create_edge(n11, n12); Edge* e18 = graph.create_edge(n12, n13); - //vg::io::save_handle_graph(&graph, "test_graph.vg"); + vg::io::save_handle_graph(&graph, "test_graph.vg"); //get the snarls IntegratedSnarlFinder snarl_finder(graph); From 934e53efac8174f69a80b3b9f62e17d704c118a5 Mon Sep 17 00:00:00 2001 From: Adam Novak Date: Fri, 20 Mar 2026 14:21:27 -0700 Subject: [PATCH 41/77] Turn off debugging and don't count bounds as children for regularity --- deps/libbdsg | 2 +- src/snarl_distance_index.cpp | 12 +++++++++++- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/deps/libbdsg b/deps/libbdsg index 46aa0941b7..efa47f2bce 160000 --- a/deps/libbdsg +++ b/deps/libbdsg @@ -1 +1 @@ -Subproject commit 46aa0941b73b809879c7c0a6a7f239cbbe6d45b8 +Subproject commit efa47f2bceb68cbc7159fb0ddced9c478f6962a6 diff --git a/src/snarl_distance_index.cpp b/src/snarl_distance_index.cpp index 0a1517b3e2..301d651cde 100644 --- a/src/snarl_distance_index.cpp +++ b/src/snarl_distance_index.cpp @@ -1,4 +1,4 @@ -#define debug_distance_indexing +//#define debug_distance_indexing //#define debug_snarl_traversal //#define debug_distances //#define debug_subgraph @@ -1655,6 +1655,16 @@ bool check_regularity(const SnarlDistanceIndex::TemporaryDistanceIndex& temp_ind }; for (const SnarlDistanceIndex::temp_record_ref_t& child_index : all_children) { + // We should only have nodes and chains as children + assert(child_index.first == SnarlDistanceIndex::TEMP_NODE + || child_index.first == SnarlDistanceIndex::TEMP_CHAIN); + if (child_index.first == SnarlDistanceIndex::TEMP_NODE + && (child_index.second == temp_snarl_record.start_node_id + || child_index.second == temp_snarl_record.end_node_id)) { + // Don't think about children for the snarl bounds now; we handle the bounds later. + continue; + } + // Have we seen the snarl start? bool saw_start = false; // Have we seen the snarl end? From 21d2bb6b6179709cffe02cc4ba437883061ffee4 Mon Sep 17 00:00:00 2001 From: Adam Novak Date: Fri, 20 Mar 2026 15:14:37 -0700 Subject: [PATCH 42/77] Set looping "distances" in distanceless index so we can tell snarls are start-start or end-end connected --- src/snarl_distance_index.cpp | 88 +++++++++++++++++++++++++++++------- 1 file changed, 72 insertions(+), 16 deletions(-) diff --git a/src/snarl_distance_index.cpp b/src/snarl_distance_index.cpp index 301d651cde..546b24e37f 100644 --- a/src/snarl_distance_index.cpp +++ b/src/snarl_distance_index.cpp @@ -494,7 +494,7 @@ SnarlDistanceIndex::TemporaryDistanceIndex make_temporary_distance_index( SnarlDistanceIndex::TemporaryDistanceIndex::TemporaryChainRecord& temp_chain_record = temp_index.get_chain(chain_index); #ifdef debug_distance_indexing assert(!temp_chain_record.is_trivial); - cerr << " At " << (temp_chain_record.is_trivial ? " trivial " : "") << " chain " << temp_index.structure_start_end_as_string(chain_index) << endl; + cerr << " At" << (temp_chain_record.is_trivial ? " trivial " : "") << "chain " << temp_index.structure_start_end_as_string(chain_index) << endl; #endif //Add the first values for the prefix sum and backwards loop vectors @@ -1061,10 +1061,6 @@ void populate_snarl_index( all_children.emplace_back(SnarlDistanceIndex::TEMP_NODE, temp_snarl_record.end_node_id); } - #ifdef debug_distance_indexing - cerr << "is_simple: " << temp_snarl_record.is_simple << endl; - #endif - if (size_limit != 0 && temp_snarl_record.node_count > size_limit) { temp_index.most_oversized_snarl_size = std::max(temp_index.most_oversized_snarl_size, temp_snarl_record.node_count); temp_index.use_oversized_snarls = true; @@ -1090,7 +1086,11 @@ void populate_snarl_index( } //Also fills in min_lenght, distance_start_start, and distance_start_end, and sets is_simple to false if snarl isn't simple populate_distance_matrix_if_needed(temp_index, snarl_index, temp_snarl_record, all_children, graph, size_limit, only_top_level_chain_distances); - } + } + +#ifdef debug_distance_indexing + cerr << "snarl " << temp_index.structure_start_end_as_string(snarl_index) << " is_simple: " << temp_snarl_record.is_simple << endl; +#endif if (temp_snarl_record.is_simple) { // If this is a simple snarl (one with only single nodes that connect to the start and end nodes), then @@ -1364,12 +1364,19 @@ void populate_distance_matrix_row(SnarlDistanceIndex::TemporaryDistanceIndex& te temp_index.get_chain(current_index).end_node_rev)); #ifdef debug_distance_indexing - cerr << " at child " << temp_index.structure_start_end_as_string(current_index) << " going " - << (current_rev ? "rev" : "fd") << " at actual node " << graph->get_id(current_end_handle) - << (graph->get_is_reverse(current_end_handle) ? "rev" : "fd") << endl; + cerr << " at child " << temp_index.structure_start_end_as_string(current_index) << " going " + << (current_rev ? "rev" : "fd") << " at actual node " << graph->get_id(current_end_handle) + << (graph->get_is_reverse(current_end_handle) ? "rev" : "fd") << endl; #endif graph->follow_edges(current_end_handle, false, [&](const handle_t& next_handle) { - if (graph->get_id(current_end_handle) == graph->get_id(next_handle)){ +#ifdef debug_distance_indexing + cerr << " see edge " << graph->get_id(current_end_handle) + << (graph->get_is_reverse(current_end_handle) ? "rev" : "fd") + << " -> " << graph->get_id(next_handle) + << (graph->get_is_reverse(next_handle) ? "rev" : "fd") << endl; +#endif + + if (graph->get_id(current_end_handle) == graph->get_id(next_handle)) { //If this loops onto the same node side then this isn't a simple snarl temp_snarl_record.is_simple = false; } else if ((current_index.first == SnarlDistanceIndex::TEMP_NODE ? current_index.second @@ -1412,8 +1419,14 @@ void populate_distance_matrix_row(SnarlDistanceIndex::TemporaryDistanceIndex& te ? node_record.rank_in_parent : temp_index.get_chain(next_index).rank_in_parent; if (next_index.first == SnarlDistanceIndex::TEMP_NODE && next_index.second == temp_snarl_record.start_node_id) { +#ifdef debug_distance_indexing + std::cerr << " edge arrived at start" << std::endl; +#endif next_rank = 0; } else if (next_index.first == SnarlDistanceIndex::TEMP_NODE && next_index.second == temp_snarl_record.end_node_id) { +#ifdef debug_distance_indexing + std::cerr << " edge arrived at end" << std::endl; +#endif next_rank = 1; } else { //If the next thing wasn't a boundary node and this was an internal node, then it isn't a simple snarl @@ -1432,26 +1445,49 @@ void populate_distance_matrix_row(SnarlDistanceIndex::TemporaryDistanceIndex& te bool start_is_boundary = !temp_snarl_record.is_root_snarl && (start_rank == 0 || start_rank == 1); bool next_is_boundary = !temp_snarl_record.is_root_snarl && (next_rank == 0 || next_rank == 1); - if (size_limit != 0 && + pair start = start_is_boundary + ? make_pair(start_rank, false) : make_pair(start_rank, !start_rev); + pair next = next_is_boundary + ? make_pair(next_rank, false) : make_pair(next_rank, next_rev); + + if (size_limit == 0 && start_is_boundary && next_is_boundary) { + // If not measuring distances, we need to use + // distance_start_start and distance_end_end as + // connectivity flags so we can still detect reversals + // within chains and recognize regular snarls. + if (start_rank == 0 && next_rank == 0) { + temp_snarl_record.distance_start_start = 0; +#ifdef debug_distance_indexing + cerr << " set loop indicator start start distance " << temp_snarl_record.distance_start_start << endl; +#endif + } else if (start_rank == 1 && next_rank == 1) { + temp_snarl_record.distance_end_end = 0; +#ifdef debug_distance_indexing + cerr << " set loop indicator end end distance " << temp_snarl_record.distance_start_start << endl; +#endif + } + } else if (size_limit != 0 && (temp_snarl_record.node_count <= size_limit || start_is_boundary || next_is_boundary)) { //If the snarl is too big, then we don't record distances between internal nodes //If we are looking at all distances or we are looking at boundaries bool added_new_distance = false; //Set the distance - pair start = start_is_boundary - ? make_pair(start_rank, false) : make_pair(start_rank, !start_rev); - pair next = next_is_boundary - ? make_pair(next_rank, false) : make_pair(next_rank, next_rev); if (start_is_boundary && next_is_boundary) { //If it is between bounds of the snarl, then the snarl stores it if (start_rank == 0 && next_rank == 0 && temp_snarl_record.distance_start_start == std::numeric_limits::max()) { temp_snarl_record.distance_start_start = current_distance; +#ifdef debug_distance_indexing + cerr << " set start start distance " << temp_snarl_record.distance_start_start << endl; +#endif added_new_distance = true; } else if (start_rank == 1 && next_rank == 1 && temp_snarl_record.distance_end_end == std::numeric_limits::max()) { temp_snarl_record.distance_end_end = current_distance; +#ifdef debug_distance_indexing + cerr << " set end end distance " << temp_snarl_record.distance_start_start << endl; +#endif added_new_distance = true; } else if (((start_rank == 0 && next_rank == 1) || (start_rank == 1 && next_rank == 0)) && temp_snarl_record.min_length == std::numeric_limits::max()){ @@ -1554,7 +1590,7 @@ void populate_distance_matrix_row(SnarlDistanceIndex::TemporaryDistanceIndex& te } } #ifdef debug_distance_indexing - cerr << " reached child " << temp_index.structure_start_end_as_string(next_index) << "going " + cerr << " reached child " << temp_index.structure_start_end_as_string(next_index) << " going " << (next_rev ? "rev" : "fd") << " with distance " << current_distance << " for ranks " << start_rank << " " << next_rank << endl; #endif }); @@ -1735,7 +1771,18 @@ bool check_regularity(const SnarlDistanceIndex::TemporaryDistanceIndex& temp_ind if (child_index.first == SnarlDistanceIndex::TEMP_CHAIN) { // If a child is a chain, check it for loops +#ifdef debug_distance_indexing + std::cerr << "Check child chain for loops." << std::endl; +#endif const SnarlDistanceIndex::TemporaryDistanceIndex::TemporaryChainRecord& temp_chain_record = temp_index.get_chain(child_index); +#ifdef debug_distance_indexing + std::cerr << "Forward loops:"; + for (auto& l : temp_chain_record.forward_loops) { + std::cerr << " " << l; + } + std::cerr << std::endl; +#endif + if (!temp_chain_record.forward_loops.empty() && temp_chain_record.forward_loops.front() != std::numeric_limits::max()) { // There's a forward loop in this child chain, so the snarl's not regular. #ifdef debug_distance_indexing @@ -1743,6 +1790,15 @@ bool check_regularity(const SnarlDistanceIndex::TemporaryDistanceIndex& temp_ind #endif return false; } + +#ifdef debug_distance_indexing + std::cerr << "Backward loops:"; + for (auto& l : temp_chain_record.backward_loops) { + std::cerr << " " << l; + } + std::cerr << std::endl; +#endif + if (!temp_chain_record.backward_loops.empty() && temp_chain_record.backward_loops.back() != std::numeric_limits::max()) { // There's a backward loop in this child chain, so the snarl's not regular. #ifdef debug_distance_indexing From a203cb6a3562cd0c12d8ca5b54bf7818ec56ae21 Mon Sep 17 00:00:00 2001 From: Adam Novak Date: Fri, 20 Mar 2026 15:39:15 -0700 Subject: [PATCH 43/77] Use libbdsg that tries not to make way too many MPHF threads --- deps/libbdsg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deps/libbdsg b/deps/libbdsg index efa47f2bce..bb713f43f6 160000 --- a/deps/libbdsg +++ b/deps/libbdsg @@ -1 +1 @@ -Subproject commit efa47f2bceb68cbc7159fb0ddced9c478f6962a6 +Subproject commit bb713f43f6e38fa57fc54948ff89c71ba65b05b0 From 07d047236987e69c0351edcec8acd1988408ac9f Mon Sep 17 00:00:00 2001 From: Adam Novak Date: Fri, 20 Mar 2026 15:55:25 -0700 Subject: [PATCH 44/77] Add another test to make sure we aren't missing reversals hiding in the middle of chains --- src/unittest/snarl_distance_index.cpp | 40 ++++++++++++++++++++++++++- 1 file changed, 39 insertions(+), 1 deletion(-) diff --git a/src/unittest/snarl_distance_index.cpp b/src/unittest/snarl_distance_index.cpp index f540c7a02b..070adcf5fc 100644 --- a/src/unittest/snarl_distance_index.cpp +++ b/src/unittest/snarl_distance_index.cpp @@ -232,7 +232,45 @@ namespace vg { } } } - TEST_CASE( "Can distance index nested chain with loop", "[snarl_distance]" ) { + TEST_CASE( "Can distance index nested chain with a loop hiding in the middle", "[snarl_distance]" ) { + bdsg::HashGraph graph; + handle_t h1 = graph.create_handle("G"); + handle_t h2 = graph.create_handle("A"); + handle_t h3 = graph.create_handle("T"); + handle_t h4 = graph.create_handle("T"); + handle_t h5 = graph.create_handle("A"); + handle_t h6 = graph.create_handle("C"); + handle_t h7 = graph.create_handle("A"); + + // Wire it up as a stick + graph.create_edge(h1, h2); + graph.create_edge(h2, h3); + graph.create_edge(h3, h4); + graph.create_edge(h4, h5); + graph.create_edge(h5, h6); + graph.create_edge(h6, h7); + + // Allow skipping a run of nodes to make a snarl with a child chain that has a few nodes in it + graph.create_edge(h1, h6); + + // Allow turning around with an edge hiding somewhere in the middle of the chain + graph.create_edge(h3, graph.flip(h3)); + + IntegratedSnarlFinder snarl_finder(graph); + + SECTION("Snarl classifications are correct") { + SECTION("Distance index") { + SnarlDistanceIndex distance_index; + fill_in_distance_index(&distance_index, &graph, &snarl_finder); + REQUIRE(!distance_index.is_regular_snarl(distance_index.get_parent(distance_index.get_parent(distance_index.get_node_net_handle(graph.get_id(h3)))))); + } SECTION("Distanceless index") { + SnarlDistanceIndex distance_index; + fill_in_distance_index(&distance_index, &graph, &snarl_finder, 0); + REQUIRE(!distance_index.is_regular_snarl(distance_index.get_parent(distance_index.get_parent(distance_index.get_node_net_handle(graph.get_id(h3)))))); + } + } + } + TEST_CASE( "Can distance index nested chain with a loop", "[snarl_distance]" ) { VG graph; From 9aa832ac90f46f5405c20d8a7a05072bbad76332 Mon Sep 17 00:00:00 2001 From: Zia Truong <194475824+electricEpilith@users.noreply.github.com> Date: Sat, 21 Mar 2026 15:22:45 -0700 Subject: [PATCH 45/77] don't build tests for sparsehash due to C++20 incompatibility Co-authored-by: Claude Sonnet 4.6 --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 855af019c2..6159297339 100644 --- a/Makefile +++ b/Makefile @@ -822,7 +822,7 @@ $(INC_DIR)/dynamic/dynamic.hpp: $(DYNAMIC_DIR)/include/dynamic/*.hpp $(DYNAMIC_D +mkdir -p $(INC_DIR)/dynamic && cp -r $(CWD)/$(DYNAMIC_DIR)/include/dynamic/* $(INC_DIR)/dynamic/ $(INC_DIR)/sparsehash/sparse_hash_map: $(wildcard $(SPARSEHASH_DIR)/**/*.cc) $(wildcard $(SPARSEHASH_DIR)/**/*.h) - +cd $(SPARSEHASH_DIR) && ./autogen.sh && LDFLAGS="$(LD_LIB_DIR_FLAGS) $(LDFLAGS)" ./configure --prefix=$(CWD) $(FILTER) && $(MAKE) $(FILTER) && $(MAKE) install + +cd $(SPARSEHASH_DIR) && ./autogen.sh && LDFLAGS="$(LD_LIB_DIR_FLAGS) $(LDFLAGS)" ./configure --prefix=$(CWD) $(FILTER) && $(MAKE) src/sparsehash/internal/sparseconfig.h $(FILTER) && $(MAKE) install-data $(FILTER) $(INC_DIR)/sparsepp/spp.h: $(wildcard $(SPARSEPP_DIR)/sparsepp/*.h) +cp -r $(SPARSEPP_DIR)/sparsepp $(INC_DIR)/ From 85b2f4493d1b7d7cb40d03fa682f845f6cf66847 Mon Sep 17 00:00:00 2001 From: Zia Truong <194475824+electricEpilith@users.noreply.github.com> Date: Sat, 21 Mar 2026 15:22:45 -0700 Subject: [PATCH 46/77] don't build tests for sparsehash due to C++20 incompatibility --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 855af019c2..6159297339 100644 --- a/Makefile +++ b/Makefile @@ -822,7 +822,7 @@ $(INC_DIR)/dynamic/dynamic.hpp: $(DYNAMIC_DIR)/include/dynamic/*.hpp $(DYNAMIC_D +mkdir -p $(INC_DIR)/dynamic && cp -r $(CWD)/$(DYNAMIC_DIR)/include/dynamic/* $(INC_DIR)/dynamic/ $(INC_DIR)/sparsehash/sparse_hash_map: $(wildcard $(SPARSEHASH_DIR)/**/*.cc) $(wildcard $(SPARSEHASH_DIR)/**/*.h) - +cd $(SPARSEHASH_DIR) && ./autogen.sh && LDFLAGS="$(LD_LIB_DIR_FLAGS) $(LDFLAGS)" ./configure --prefix=$(CWD) $(FILTER) && $(MAKE) $(FILTER) && $(MAKE) install + +cd $(SPARSEHASH_DIR) && ./autogen.sh && LDFLAGS="$(LD_LIB_DIR_FLAGS) $(LDFLAGS)" ./configure --prefix=$(CWD) $(FILTER) && $(MAKE) src/sparsehash/internal/sparseconfig.h $(FILTER) && $(MAKE) install-data $(FILTER) $(INC_DIR)/sparsepp/spp.h: $(wildcard $(SPARSEPP_DIR)/sparsepp/*.h) +cp -r $(SPARSEPP_DIR)/sparsepp $(INC_DIR)/ From 180dbf0c3844f949caefbc7e395475b496a65bd3 Mon Sep 17 00:00:00 2001 From: Zia Truong <194475824+electricEpilith@users.noreply.github.com> Date: Mon, 23 Mar 2026 18:28:34 -0700 Subject: [PATCH 47/77] Parallelize cache_payloads and re-preload distance index before it cache_payloads was single-threaded despite the -t flag; with 164M nodes on an HPRC graph it hung for hours. Two fixes: 1. Pass `true` to for_each_handle to enable OpenMP parallelism; guard the non-thread-safe writes (oversized_zipcodes vector and node_id_to_payload map) with named omp critical sections. 2. Call distance_index->preload(true) immediately before cache_payloads in build_minimizer_index. find_frequent_kmers runs for ~3300 s before this point and evicts the mmap'd index pages, causing a page fault on every snarl-tree lookup in fill_in_zipcode_from_pos. Reloading here ensures the index is warm when the parallel loop starts. Also add a depth guard (abort at >10000) in fill_in_zipcode_from_pos to catch any future infinite loops in the snarl tree traversal. Also use distance_index.get_snarl_child_count() (O(1) record read) instead of for_each_child iteration in get_regular/irregular_snarl_code. Co-Authored-By: Claude Sonnet 4.6 --- src/gbwtgraph_helper.cpp | 33 ++++++++++++++++++++++++++------- src/zip_code.cpp | 23 ++++++++++------------- 2 files changed, 36 insertions(+), 20 deletions(-) diff --git a/src/gbwtgraph_helper.cpp b/src/gbwtgraph_helper.cpp index 1153d4a135..75e5418f94 100644 --- a/src/gbwtgraph_helper.cpp +++ b/src/gbwtgraph_helper.cpp @@ -442,22 +442,37 @@ void cache_payloads( const handlegraph::HandleGraph* graph_ptr = (const handlegraph::HandleGraph*) &gbz.graph; + double total_zipcode_time = 0.0, total_decoder_time = 0.0; + uint64_t node_count = 0; gbz.graph.for_each_handle([&](const handle_t& handle) { nid_t node_id = gbz.graph.get_id(handle); - ZipCode zipcode; pos_t pos = make_pos_t(node_id, false, 0); - zipcode.fill_in_zipcode_from_pos(distance_index, pos, true, graph_ptr); + ZipCode zipcode; + zipcode.fill_in_zipcode_from_pos(distance_index, pos, false, graph_ptr); + zipcode.fill_in_full_decoder(); + node_count++; + if (node_count % 10000 == 0) { + double telapsed = gbwt::readTimer() - start; + std::cerr << " Cached " << node_count << " nodes in " << telapsed << "s" << std::endl; + } + payload_t payload = zipcode.get_payload_from_zip(); if (payload == MIPayload::NO_CODE && oversized_zipcodes != nullptr) { // The zipcode is too large for the payload field. // Add it to the oversized zipcode list. - zipcode.fill_in_full_decoder(); - size_t offset = oversized_zipcodes->size(); - oversized_zipcodes->emplace_back(zipcode); + size_t offset; + #pragma omp critical (cache_payloads_zipcodes) + { + offset = oversized_zipcodes->size(); + oversized_zipcodes->emplace_back(zipcode); + } payload = { 0, offset }; } - node_id_to_payload.emplace(node_id, payload); - }); + #pragma omp critical (cache_payloads_map) + { + node_id_to_payload.emplace(node_id, payload); + } + }, true); if (progress) { double seconds = gbwt::readTimer() - start; @@ -521,6 +536,10 @@ gbwtgraph::DefaultMinimizerIndex build_minimizer_index( // A zipcode only depends on the node id. vg::hash_map node_id_to_payload; node_id_to_payload.reserve(gbz.graph.max_node_id() - gbz.graph.min_node_id()); + // Re-preload the distance index: find_frequent_kmers runs for a long time before this + // point and evicts the mmap'd index pages from the OS page cache, causing cache_payloads + // to page-fault on every node. Reloading here ensures the index is warm. + distance_index->preload(true); cache_payloads(gbz, *distance_index, node_id_to_payload, oversized_zipcodes, params.progress); auto get_payload = [&](const pos_t& pos) -> const code_type* { diff --git a/src/zip_code.cpp b/src/zip_code.cpp index 14b97b102a..1da78e1a98 100644 --- a/src/zip_code.cpp +++ b/src/zip_code.cpp @@ -14,12 +14,17 @@ void ZipCode::fill_in_zipcode_from_pos(const SnarlDistanceIndex& distance_index, net_handle_t current_handle = distance_index.get_node_net_handle(id(pos)); //Put all ancestors of the node in a vector, starting from the node, and not including the root + size_t depth = 0; while (!distance_index.is_root(current_handle)) { ancestors.emplace_back(distance_index.start_end_traversal_of(current_handle)); current_handle = distance_index.get_parent(current_handle); + if (++depth > 10000) { + std::cerr << "[fill_in_zipcode_from_pos] ERROR: ancestor loop exceeded depth 10000 at node " + << id(pos) << " — likely infinite loop in snarl tree" << std::endl; + std::abort(); + } } - //Now add the root-level snarl or chain if (distance_index.is_root_snarl(current_handle)) { //First thing is a snarl, so add the snarl's connected component number @@ -1064,12 +1069,8 @@ ZipCode::snarl_code_t ZipCode::get_regular_snarl_code(const net_handle_t& snarl, //Tag to say that it's a regular snarl snarl_code.set_code_type(1); - //The number of children - size_t child_count = 0; - distance_index.for_each_child(snarl, [&] (const net_handle_t& child) { - child_count++; - }); - snarl_code.set_child_count(child_count); + //The number of children — read directly from the stored record (O(1)) rather than iterating + snarl_code.set_child_count(distance_index.get_snarl_child_count(snarl)); //Chain prefix sum value for the start of the snarl, which is the prefix sum of the start node + length of the start node net_handle_t start_node = distance_index.get_node_from_sentinel(distance_index.get_bound(snarl, false, false)); @@ -1099,12 +1100,8 @@ ZipCode::snarl_code_t ZipCode::get_irregular_snarl_code(const net_handle_t& snar //Tag to say that it's an irregular snarl snarl_code.set_code_type(distance_index.is_dag(snarl) ? 0 : 2); - //The number of children - size_t child_count = 0; - distance_index.for_each_child(snarl, [&] (const net_handle_t& child) { - child_count++; - }); - snarl_code.set_child_count(child_count); + //The number of children — read directly from the stored record (O(1)) rather than iterating + snarl_code.set_child_count(distance_index.get_snarl_child_count(snarl)); //Chain prefix sum value for the start of the snarl, which is the prefix sum of the start node + length of the start node net_handle_t start_node = distance_index.get_node_from_sentinel(distance_index.get_bound(snarl, false, false)); From b8d310d81daaa3b7977d85e57861859ddea9f131 Mon Sep 17 00:00:00 2001 From: Adam Novak Date: Tue, 24 Mar 2026 10:59:42 -0400 Subject: [PATCH 48/77] Atomic-ize progress and remove uninformative comment text --- src/gbwtgraph_helper.cpp | 6 +++--- src/zip_code.cpp | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/gbwtgraph_helper.cpp b/src/gbwtgraph_helper.cpp index 75e5418f94..4ec276440a 100644 --- a/src/gbwtgraph_helper.cpp +++ b/src/gbwtgraph_helper.cpp @@ -443,16 +443,16 @@ void cache_payloads( const handlegraph::HandleGraph* graph_ptr = (const handlegraph::HandleGraph*) &gbz.graph; double total_zipcode_time = 0.0, total_decoder_time = 0.0; - uint64_t node_count = 0; + std::atomic node_count = 0; gbz.graph.for_each_handle([&](const handle_t& handle) { nid_t node_id = gbz.graph.get_id(handle); pos_t pos = make_pos_t(node_id, false, 0); ZipCode zipcode; zipcode.fill_in_zipcode_from_pos(distance_index, pos, false, graph_ptr); zipcode.fill_in_full_decoder(); - node_count++; - if (node_count % 10000 == 0) { + if (++node_count % 10000 == 0 && progress) { double telapsed = gbwt::readTimer() - start; + #pragma omp critical (cerr) std::cerr << " Cached " << node_count << " nodes in " << telapsed << "s" << std::endl; } diff --git a/src/zip_code.cpp b/src/zip_code.cpp index 1da78e1a98..d5ed5df48d 100644 --- a/src/zip_code.cpp +++ b/src/zip_code.cpp @@ -1069,7 +1069,7 @@ ZipCode::snarl_code_t ZipCode::get_regular_snarl_code(const net_handle_t& snarl, //Tag to say that it's a regular snarl snarl_code.set_code_type(1); - //The number of children — read directly from the stored record (O(1)) rather than iterating + //The number of children snarl_code.set_child_count(distance_index.get_snarl_child_count(snarl)); //Chain prefix sum value for the start of the snarl, which is the prefix sum of the start node + length of the start node @@ -1100,7 +1100,7 @@ ZipCode::snarl_code_t ZipCode::get_irregular_snarl_code(const net_handle_t& snar //Tag to say that it's an irregular snarl snarl_code.set_code_type(distance_index.is_dag(snarl) ? 0 : 2); - //The number of children — read directly from the stored record (O(1)) rather than iterating + //The number of children snarl_code.set_child_count(distance_index.get_snarl_child_count(snarl)); //Chain prefix sum value for the start of the snarl, which is the prefix sum of the start node + length of the start node From 0281b81a489ad68343c2baeecefe2533db206ea7 Mon Sep 17 00:00:00 2001 From: Adam Novak Date: Tue, 24 Mar 2026 11:07:55 -0400 Subject: [PATCH 49/77] Replace snarl tree depth limit with fixed point check --- src/zip_code.cpp | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/src/zip_code.cpp b/src/zip_code.cpp index d5ed5df48d..9e4a610f27 100644 --- a/src/zip_code.cpp +++ b/src/zip_code.cpp @@ -14,15 +14,11 @@ void ZipCode::fill_in_zipcode_from_pos(const SnarlDistanceIndex& distance_index, net_handle_t current_handle = distance_index.get_node_net_handle(id(pos)); //Put all ancestors of the node in a vector, starting from the node, and not including the root - size_t depth = 0; while (!distance_index.is_root(current_handle)) { ancestors.emplace_back(distance_index.start_end_traversal_of(current_handle)); - current_handle = distance_index.get_parent(current_handle); - if (++depth > 10000) { - std::cerr << "[fill_in_zipcode_from_pos] ERROR: ancestor loop exceeded depth 10000 at node " - << id(pos) << " — likely infinite loop in snarl tree" << std::endl; - std::abort(); - } + net_handle_t parent_handle = distance_index.get_parent(current_handle); + crash_unless(parent_handle != current_handle, "net handle is its own parent"); + current_handle = parent_handle; } //Now add the root-level snarl or chain From 4422b08800d42175ad3e58c432a34701aeafbc69 Mon Sep 17 00:00:00 2001 From: Adam Novak Date: Tue, 24 Mar 2026 11:17:52 -0400 Subject: [PATCH 50/77] Preload distance index only once --- src/gbwtgraph_helper.cpp | 11 ++++++++--- src/subcommand/minimizer_main.cpp | 5 +++-- 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/src/gbwtgraph_helper.cpp b/src/gbwtgraph_helper.cpp index 4ec276440a..3521f4d6f8 100644 --- a/src/gbwtgraph_helper.cpp +++ b/src/gbwtgraph_helper.cpp @@ -536,9 +536,14 @@ gbwtgraph::DefaultMinimizerIndex build_minimizer_index( // A zipcode only depends on the node id. vg::hash_map node_id_to_payload; node_id_to_payload.reserve(gbz.graph.max_node_id() - gbz.graph.min_node_id()); - // Re-preload the distance index: find_frequent_kmers runs for a long time before this - // point and evicts the mmap'd index pages from the OS page cache, causing cache_payloads - // to page-fault on every node. Reloading here ensures the index is warm. + // Preload the distance index right before we use it. + // find_frequent_kmers uses a lot of memory/IO scanning the whole graph + // and might evict the mmap'd index pages from the OS page cache, + // causing cache_payloads to page-fault on every node. So we preload + // after kmer counting to ensure the index is warm. + if (params.progress) { + std::cerr << "Preloading distance index"; + } distance_index->preload(true); cache_payloads(gbz, *distance_index, node_id_to_payload, oversized_zipcodes, params.progress); diff --git a/src/subcommand/minimizer_main.cpp b/src/subcommand/minimizer_main.cpp index 8635df2a97..eeb73a850c 100644 --- a/src/subcommand/minimizer_main.cpp +++ b/src/subcommand/minimizer_main.cpp @@ -88,10 +88,11 @@ int main_minimizer(int argc, char** argv) { if (!config.distance_name.empty()) { // new distance index if (config.progress) { - logger.info() << "Loading SnarlDistanceIndex from " << config.distance_name << std::endl; + logger.info() << "Opening SnarlDistanceIndex at " << config.distance_name << std::endl; } distance_index = vg::io::VPKG::load_one(config.distance_name); - distance_index->preload(true); + // Note that we don't fault in the index until we're actually about to + // use it, or it might get paged out again. } ZipCodeCollection oversized_zipcodes; From aa2d827f95b197da19e786c3fca5d7d1971e904e Mon Sep 17 00:00:00 2001 From: Adam Novak Date: Tue, 24 Mar 2026 11:38:40 -0400 Subject: [PATCH 51/77] Remove extra argument --- src/zip_code.cpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/zip_code.cpp b/src/zip_code.cpp index 9e4a610f27..051602443f 100644 --- a/src/zip_code.cpp +++ b/src/zip_code.cpp @@ -1,3 +1,5 @@ +#include "crash.hpp" + #include "zip_code.hpp" //#define DEBUG_ZIPCODE @@ -17,7 +19,7 @@ void ZipCode::fill_in_zipcode_from_pos(const SnarlDistanceIndex& distance_index, while (!distance_index.is_root(current_handle)) { ancestors.emplace_back(distance_index.start_end_traversal_of(current_handle)); net_handle_t parent_handle = distance_index.get_parent(current_handle); - crash_unless(parent_handle != current_handle, "net handle is its own parent"); + crash_unless(parent_handle != current_handle); current_handle = parent_handle; } From 96b57ee1f8e2cafe21c0deccdd833996c601291c Mon Sep 17 00:00:00 2001 From: Adam Novak Date: Tue, 24 Mar 2026 13:09:07 -0400 Subject: [PATCH 52/77] Use libbdsg that should define child snarl count function --- deps/libbdsg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deps/libbdsg b/deps/libbdsg index bb713f43f6..00327bd7c9 160000 --- a/deps/libbdsg +++ b/deps/libbdsg @@ -1 +1 @@ -Subproject commit bb713f43f6e38fa57fc54948ff89c71ba65b05b0 +Subproject commit 00327bd7c92f61613265478c7bc44d2962f0cddf From 3c27df0f9ed44d84032ca63a758ba585cb65dc5a Mon Sep 17 00:00:00 2001 From: Adam Novak Date: Tue, 24 Mar 2026 18:35:43 -0400 Subject: [PATCH 53/77] Regular-ify simple snarls --- deps/libbdsg | 2 +- src/snarl_distance_index.cpp | 10 ++++++---- src/unittest/zip_code.cpp | 4 ++++ 3 files changed, 11 insertions(+), 5 deletions(-) diff --git a/deps/libbdsg b/deps/libbdsg index 00327bd7c9..72eefa7dab 160000 --- a/deps/libbdsg +++ b/deps/libbdsg @@ -1 +1 @@ -Subproject commit 00327bd7c92f61613265478c7bc44d2962f0cddf +Subproject commit 72eefa7dab8791c180bcb6238f67f497507c59a3 diff --git a/src/snarl_distance_index.cpp b/src/snarl_distance_index.cpp index 546b24e37f..fb20c451e0 100644 --- a/src/snarl_distance_index.cpp +++ b/src/snarl_distance_index.cpp @@ -814,10 +814,12 @@ static void populate_hub_labeling(SnarlDistanceIndex::TemporaryDistanceIndex& te /** * Determine if a snarl is regular or not. * - * A regular snarl is a snarl that, while not simple, consists of only nodes or + * A regular snarl is a snarl that consists of only nodes or * chains connected to the start and end, without any connections between * multiple children, or any way to turn around. There may be an edge directly * across. + * + * A simple snarl is always regular. */ static bool check_regularity(const SnarlDistanceIndex::TemporaryDistanceIndex& temp_index, const SnarlDistanceIndex::temp_record_ref_t& snarl_index, const SnarlDistanceIndex::TemporaryDistanceIndex::TemporarySnarlRecord& temp_snarl_record, const vector& all_children, const HandleGraph* graph); @@ -1661,11 +1663,11 @@ bool check_regularity(const SnarlDistanceIndex::TemporaryDistanceIndex& temp_ind return false; } if (temp_snarl_record.is_simple) { - // Simple snarls can't be regular because simple is more specific and useful. + // Simple snarls are always also regular. #ifdef debug_distance_indexing - std::cerr << "Snarl is not regular because it is simple." << std::endl; + std::cerr << "Snarl is regular because it is simple." << std::endl; #endif - return false; + return true; } // Get the snarl boundary nodes, facing out diff --git a/src/unittest/zip_code.cpp b/src/unittest/zip_code.cpp index dc3255e984..1d0a2c39c7 100644 --- a/src/unittest/zip_code.cpp +++ b/src/unittest/zip_code.cpp @@ -117,6 +117,10 @@ using namespace std; bool chain_is_reversed = distance_index.is_reversed_in_parent( distance_index.get_node_net_handle(n1->id())); + // Node 4 is in snarl 3 to 6 which should be regular. + // The zip codes are going to encode this so it had better be true. + REQUIRE(distance_index.is_regular_snarl(distance_index.get_parent(distance_index.get_parent(distance_index.get_node_net_handle(n4->id()))))); + SECTION ("zip code for node on top-level chain") { ZipCode zipcode; zipcode.fill_in_zipcode_from_pos(distance_index, make_pos_t(n1->id(), 0, false)); From 644b900e1d035de648becc1e8b00ee2087340eb8 Mon Sep 17 00:00:00 2001 From: Zia Truong <194475824+electricEpilith@users.noreply.github.com> Date: Mon, 30 Mar 2026 14:36:41 -0700 Subject: [PATCH 54/77] move libbdsg up --- deps/libbdsg | 2 +- src/snarl_distance_index.cpp | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/deps/libbdsg b/deps/libbdsg index 0be8fdadfb..5acf1f4d68 160000 --- a/deps/libbdsg +++ b/deps/libbdsg @@ -1 +1 @@ -Subproject commit 0be8fdadfb4de42c6d64b628187167d518754982 +Subproject commit 5acf1f4d68aad0c97b7e22414bb441c9fe24e9d4 diff --git a/src/snarl_distance_index.cpp b/src/snarl_distance_index.cpp index 6a5d7e934e..9cc614a700 100644 --- a/src/snarl_distance_index.cpp +++ b/src/snarl_distance_index.cpp @@ -1269,7 +1269,7 @@ void populate_distance_matrix_if_needed(SnarlDistanceIndex::TemporaryDistanceInd }); if (!has_edges) { temp_index.get_node(node_index).is_tip = true; - temp_snarl_record.tippy_child_ranks.insert(rank); + temp_snarl_record.tippy_child_ranks.emplace(rank, false); temp_snarl_record.is_simple=false; //It is a tip so this isn't simple snarl } has_edges = false; @@ -1278,7 +1278,7 @@ void populate_distance_matrix_if_needed(SnarlDistanceIndex::TemporaryDistanceInd }); if (!has_edges) { temp_index.get_node(node_index).is_tip = true; - temp_snarl_record.tippy_child_ranks.insert(rank); + temp_snarl_record.tippy_child_ranks.emplace(rank, true); temp_snarl_record.is_simple=false; //It is a tip so this isn't simple snarl } } else if (start_index.first == SnarlDistanceIndex::TEMP_CHAIN && !temp_index.get_chain(start_index).is_trivial) { From 44321a3c7d3b8897f7de73e8aa7ed2bbdba86025 Mon Sep 17 00:00:00 2001 From: Zia Truong <194475824+electricEpilith@users.noreply.github.com> Date: Mon, 30 Mar 2026 15:45:22 -0700 Subject: [PATCH 55/77] added back (more) preloading to speed minimizer back up passes all tests run by `make test` Co-Authored-By: Claude Sonnet 4.6 --- src/gbwtgraph_helper.cpp | 11 ++++++----- src/subcommand/minimizer_main.cpp | 11 ++++++++--- 2 files changed, 14 insertions(+), 8 deletions(-) diff --git a/src/gbwtgraph_helper.cpp b/src/gbwtgraph_helper.cpp index 3521f4d6f8..2056dd0006 100644 --- a/src/gbwtgraph_helper.cpp +++ b/src/gbwtgraph_helper.cpp @@ -536,11 +536,12 @@ gbwtgraph::DefaultMinimizerIndex build_minimizer_index( // A zipcode only depends on the node id. vg::hash_map node_id_to_payload; node_id_to_payload.reserve(gbz.graph.max_node_id() - gbz.graph.min_node_id()); - // Preload the distance index right before we use it. - // find_frequent_kmers uses a lot of memory/IO scanning the whole graph - // and might evict the mmap'd index pages from the OS page cache, - // causing cache_payloads to page-fault on every node. So we preload - // after kmer counting to ensure the index is warm. + // Re-preload the distance index right before use. find_frequent_kmers + // runs for a long time and may evict the mmap'd index pages from the OS + // page cache. We also preload eagerly right after loading the index (in + // minimizer_main.cpp) so the kernel treats those pages as recently-used; + // together the two preloads prevent cache_payloads from page-faulting on + // every node under the memory pressure of 32 parallel threads. if (params.progress) { std::cerr << "Preloading distance index"; } diff --git a/src/subcommand/minimizer_main.cpp b/src/subcommand/minimizer_main.cpp index eeb73a850c..20e83b368e 100644 --- a/src/subcommand/minimizer_main.cpp +++ b/src/subcommand/minimizer_main.cpp @@ -88,11 +88,16 @@ int main_minimizer(int argc, char** argv) { if (!config.distance_name.empty()) { // new distance index if (config.progress) { - logger.info() << "Opening SnarlDistanceIndex at " << config.distance_name << std::endl; + logger.info() << "Loading SnarlDistanceIndex from " << config.distance_name << std::endl; } distance_index = vg::io::VPKG::load_one(config.distance_name); - // Note that we don't fault in the index until we're actually about to - // use it, or it might get paged out again. + // Preload the index eagerly to establish it as recently-used in the OS + // page cache. Even though kmer counting may evict some pages, we + // re-preload right before cache_payloads. The double-preload is + // necessary: a single preload just before cache_payloads isn't enough + // to keep the index resident under the memory pressure of 32 parallel + // threads and the remaining in-memory data structures. + distance_index->preload(true); } ZipCodeCollection oversized_zipcodes; From 0c5c0d919fabf482b9e0a02a4d9878a09e89db3e Mon Sep 17 00:00:00 2001 From: Zia Truong <194475824+electricEpilith@users.noreply.github.com> Date: Mon, 30 Mar 2026 20:56:20 -0700 Subject: [PATCH 56/77] update oldest-supported-compiler-job, upgrade gcc requirement to 10 to support -std=c++20 --- .gitlab-ci.yml | 2 +- README.md | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 143a84b419..b73794f0a3 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -158,7 +158,7 @@ oldest-supported-compiler-job: GIT_SUBMODULE_STRATEGY: none # DO NOT change this version number without updating the README to reflect # the requirement bump. - COMPILER_VERSION: 9 + COMPILER_VERSION: 10 # We define one job to do the Docker container build diff --git a/README.md b/README.md index a3e1d5e4cd..15ab9b239e 100644 --- a/README.md +++ b/README.md @@ -44,6 +44,8 @@ We maintain a support forum on biostars: https://www.biostars.org/tag/vg/ ## Installation +*Update:* GCC version 10 or higher now required for those compiling from source. + ### Download Releases The easiest way to get vg is to download one of our release builds for Linux. We have a 6-week release cadence, so our builds are never too far out of date. From 992c1cc6c16599f811006ae8b7e54e068339dea8 Mon Sep 17 00:00:00 2001 From: Zia Truong <194475824+electricEpilith@users.noreply.github.com> Date: Mon, 30 Mar 2026 21:04:51 -0700 Subject: [PATCH 57/77] edit correct place for GCC version notice --- README.md | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/README.md b/README.md index 15ab9b239e..2c616f69fe 100644 --- a/README.md +++ b/README.md @@ -44,8 +44,6 @@ We maintain a support forum on biostars: https://www.biostars.org/tag/vg/ ## Installation -*Update:* GCC version 10 or higher now required for those compiling from source. - ### Download Releases The easiest way to get vg is to download one of our release builds for Linux. We have a 6-week release cadence, so our builds are never too far out of date. @@ -95,7 +93,7 @@ On other distros, or if you do not have root access, you will need to perform th liblzma-dev liblz4-dev libffi-dev libcairo-dev libboost-all-dev \ libzstd-dev pybind11-dev python3-pybind11 libssl-dev kmc -At present, you will need GCC version 9 or greater, with support for C++17, to compile vg. (Check your version with `gcc --version`.) GCC up to 11.4.0 is supported. +At present, you will need GCC version 10 or greater, with support for C++20, to compile vg. (Check your version with `gcc --version`.) GCC up to 11.4.0 is supported. Other libraries may be required. Please report any build difficulties. From 7569cc1e3e4f6c20febcab5198f99fac150f01ca Mon Sep 17 00:00:00 2001 From: Zia <194475824+electricEpilith@users.noreply.github.com> Date: Thu, 2 Apr 2026 11:21:50 -0700 Subject: [PATCH 58/77] move up libbdsg to upgrade snarl distance index version number --- deps/libbdsg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deps/libbdsg b/deps/libbdsg index 5acf1f4d68..ea70e5575b 160000 --- a/deps/libbdsg +++ b/deps/libbdsg @@ -1 +1 @@ -Subproject commit 5acf1f4d68aad0c97b7e22414bb441c9fe24e9d4 +Subproject commit ea70e5575b37afe3866b217ba87e362542635307 From b81e331a8d4d7b19286530e521614e587e17ec73 Mon Sep 17 00:00:00 2001 From: Zia <194475824+electricEpilith@users.noreply.github.com> Date: Fri, 3 Apr 2026 09:39:37 -0700 Subject: [PATCH 59/77] add (a substantial amount of) instrumentation for vg giraffe planned out by Claude Opus 4.6 Co-Authored-By: Claude Sonnet 4.6 --- src/funnel.cpp | 4 + src/funnel.hpp | 4 + src/giraffe_stats.cpp | 200 +++++++++++++++++++++++++++ src/giraffe_stats.hpp | 73 ++++++++++ src/minimizer_mapper.cpp | 17 ++- src/minimizer_mapper.hpp | 7 + src/minimizer_mapper_from_chains.cpp | 17 ++- src/subcommand/giraffe_main.cpp | 29 +++- 8 files changed, 343 insertions(+), 8 deletions(-) create mode 100644 src/giraffe_stats.cpp create mode 100644 src/giraffe_stats.hpp diff --git a/src/funnel.cpp b/src/funnel.cpp index ebf796b3aa..06cec8fe9d 100644 --- a/src/funnel.cpp +++ b/src/funnel.cpp @@ -428,6 +428,10 @@ size_t Funnel::latest() const { return stages.back().items.size() - 1; } +double Funnel::total_seconds() const { + return chrono::duration_cast>(stop_time - start_time).count(); +} + void Funnel::for_each_stage(const function&, const vector&, const vector&, const double&, const std::unordered_map&)>& callback) const { for (auto& stage : stages) { // Make a vector of item sizes diff --git a/src/funnel.hpp b/src/funnel.hpp index 7c0add6a4d..7a08d524d2 100644 --- a/src/funnel.hpp +++ b/src/funnel.hpp @@ -223,6 +223,10 @@ class Funnel { /// Get the index of the most recent item created in the current stage. size_t latest() const; + + /// Get the total elapsed seconds from start() to stop(). + /// Only valid after stop() has been called. + double total_seconds() const; /// Call the given callback with stage name, a vector of result item sizes /// at that stage, a vector of correct item scores at that stage (if any), diff --git a/src/giraffe_stats.cpp b/src/giraffe_stats.cpp new file mode 100644 index 0000000000..3e0b5129bc --- /dev/null +++ b/src/giraffe_stats.cpp @@ -0,0 +1,200 @@ +/** + * \file giraffe_stats.cpp + */ + +#include "giraffe_stats.hpp" + +#include +#include +#include +#include +#include +#include + +namespace vg { + +GiraffeStats::GiraffeStats(size_t thread_count, double slow_threshold_s) + : per_thread(thread_count), slow_threshold_s(slow_threshold_s) {} + +void GiraffeStats::record_read(const Funnel& funnel, const std::string& read_name) { + int tid = omp_get_thread_num(); + ThreadData& td = per_thread.at(tid); + + double total = funnel.total_seconds(); + td.read_durations.push_back(total); + + bool is_slow = slow_threshold_s > 0.0 && total > slow_threshold_s; + if (is_slow) { + td.slow_read_count++; + } + + // Accumulate per-stage data and optionally build the slow-read message. + std::ostringstream slow_msg; + if (is_slow) { + slow_msg << "warning[vg::Giraffe]: Slow read \"" << read_name + << "\" took " << std::fixed << std::setprecision(3) + << total << "s:\n"; + } + + funnel.for_each_stage([&](const std::string& stage, + const std::vector& result_sizes, + const std::vector& /*correct_scores*/, + const std::vector& /*noncorrect_scores*/, + const double& duration, + const std::unordered_map& sub_durations) { + // Record into per-thread storage. + if (!td.stage_durations.count(stage)) { + td.stage_order.push_back(stage); + } + td.stage_durations[stage].push_back(duration); + td.stage_item_counts[stage].push_back(result_sizes.size()); + + for (auto& kv : sub_durations) { + std::string key = stage + "/" + kv.first; + td.substage_durations[key].push_back(kv.second); + } + + if (is_slow) { + slow_msg << " " << stage << ": " + << std::fixed << std::setprecision(3) << duration << "s" + << " (" << result_sizes.size() << " items)\n"; + for (auto& kv : sub_durations) { + slow_msg << " " << kv.first << ": " + << std::fixed << std::setprecision(3) << kv.second << "s\n"; + } + } + }); + + if (is_slow) { + #pragma omp critical (cerr) + std::cerr << slow_msg.str() << std::flush; + } +} + +double GiraffeStats::percentile(const std::vector& sorted, double p) { + if (sorted.empty()) return 0.0; + if (sorted.size() == 1) return sorted[0]; + double idx = p * (sorted.size() - 1); + size_t lo = (size_t)idx; + size_t hi = lo + 1; + if (hi >= sorted.size()) return sorted.back(); + double frac = idx - lo; + return sorted[lo] * (1.0 - frac) + sorted[hi] * frac; +} + +void GiraffeStats::print_summary(std::ostream& out) const { + // Merge all thread data. + // Use the stage order from thread 0 (or whichever thread saw stages first), + // then append any stages seen by other threads. + std::vector stage_order; + std::unordered_map> stage_durations; + std::unordered_map> stage_item_counts; + std::unordered_map> substage_durations; + std::vector read_durations; + size_t slow_read_count = 0; + size_t total_reads = 0; + + for (auto& td : per_thread) { + // Merge stage order (preserve first-seen ordering). + for (auto& s : td.stage_order) { + if (!stage_durations.count(s)) { + stage_order.push_back(s); + } + } + for (auto& kv : td.stage_durations) { + auto& dst = stage_durations[kv.first]; + dst.insert(dst.end(), kv.second.begin(), kv.second.end()); + } + for (auto& kv : td.stage_item_counts) { + auto& dst = stage_item_counts[kv.first]; + dst.insert(dst.end(), kv.second.begin(), kv.second.end()); + } + for (auto& kv : td.substage_durations) { + auto& dst = substage_durations[kv.first]; + dst.insert(dst.end(), kv.second.begin(), kv.second.end()); + } + read_durations.insert(read_durations.end(), + td.read_durations.begin(), td.read_durations.end()); + slow_read_count += td.slow_read_count; + total_reads += td.read_durations.size(); + } + + if (total_reads == 0) return; + + // Sort per-read durations for percentiles. + std::vector sorted_reads = read_durations; + std::sort(sorted_reads.begin(), sorted_reads.end()); + + // Print header. + out << "\n=== Giraffe Per-Stage Timing (" << total_reads << " reads) ===\n"; + out << std::left + << std::setw(28) << "Stage" + << std::right + << std::setw(10) << "Mean(ms)" + << std::setw(10) << "P50(ms)" + << std::setw(10) << "P95(ms)" + << std::setw(10) << "P99(ms)" + << std::setw(10) << "Max(ms)" + << std::setw(12) << "MeanItems" + << std::setw(10) << "MaxItems" + << "\n"; + + // Helper to print one row. + auto print_row = [&](const std::string& label, std::vector& durs, + const std::vector* items) { + std::sort(durs.begin(), durs.end()); + double mean_ms = 1000.0 * std::accumulate(durs.begin(), durs.end(), 0.0) / durs.size(); + double p50_ms = 1000.0 * percentile(durs, 0.50); + double p95_ms = 1000.0 * percentile(durs, 0.95); + double p99_ms = 1000.0 * percentile(durs, 0.99); + double max_ms = 1000.0 * durs.back(); + + out << std::left << std::setw(28) << label << std::right + << std::fixed << std::setprecision(2) + << std::setw(10) << mean_ms + << std::setw(10) << p50_ms + << std::setw(10) << p95_ms + << std::setw(10) << p99_ms + << std::setw(10) << max_ms; + + if (items && !items->empty()) { + double mean_items = (double)std::accumulate(items->begin(), items->end(), (size_t)0) / items->size(); + size_t max_items = *std::max_element(items->begin(), items->end()); + out << std::setw(12) << std::setprecision(1) << mean_items + << std::setw(10) << max_items; + } + out << "\n"; + }; + + for (auto& stage : stage_order) { + print_row(stage, stage_durations[stage], &stage_item_counts[stage]); + + // Print any substages for this stage, indented. + for (auto& kv : substage_durations) { + // Key format is "stage/substage". + size_t slash = kv.first.find('/'); + if (slash == std::string::npos) continue; + if (kv.first.substr(0, slash) != stage) continue; + std::string substage_label = " " + kv.first.substr(slash + 1); + print_row(substage_label, kv.second, nullptr); + } + } + + // Total read timing. + double mean_ms = 1000.0 * std::accumulate(read_durations.begin(), read_durations.end(), 0.0) / total_reads; + double p99_ms = 1000.0 * percentile(sorted_reads, 0.99); + double max_ms = 1000.0 * sorted_reads.back(); + + out << "\nTotal per-read: mean=" << std::fixed << std::setprecision(2) << mean_ms + << "ms, P99=" << p99_ms << "ms, max=" << max_ms << "ms\n"; + + if (slow_threshold_s > 0.0) { + out << "Slow reads (>" << slow_threshold_s << "s): " + << slow_read_count << " / " << total_reads + << " (" << std::fixed << std::setprecision(3) + << 100.0 * slow_read_count / total_reads << "%)\n"; + } + out << std::flush; +} + +} // namespace vg diff --git a/src/giraffe_stats.hpp b/src/giraffe_stats.hpp new file mode 100644 index 0000000000..bb3316ed97 --- /dev/null +++ b/src/giraffe_stats.hpp @@ -0,0 +1,73 @@ +#ifndef VG_GIRAFFE_STATS_HPP_INCLUDED +#define VG_GIRAFFE_STATS_HPP_INCLUDED + +/** + * \file giraffe_stats.hpp + * Aggregate per-stage timing statistics across reads for the Giraffe mapper. + */ + +#include "funnel.hpp" + +#include +#include +#include +#include + +namespace vg { + +using namespace std; + +/** + * Thread-safe aggregate statistics collector for the Giraffe mapper. + * + * Each thread pushes data into its own ThreadData slot (no locking on the hot + * path). Summary statistics are computed and printed after all reads are + * mapped. + * + * Also logs a per-stage breakdown to stderr for any individual read whose + * total mapping time exceeds a configurable threshold. + */ +class GiraffeStats { +public: + GiraffeStats(size_t thread_count, double slow_threshold_s); + + /** + * Record timing/item data from one read's Funnel. + * Must be called after funnel.stop(). + * Uses omp_get_thread_num() to select the per-thread slot. + */ + void record_read(const Funnel& funnel, const std::string& read_name); + + /** + * Print a summary table of per-stage timing percentiles to out. + * Thread-safe to call after all record_read() calls are done. + */ + void print_summary(std::ostream& out) const; + +private: + struct ThreadData { + // Per-stage accumulated durations and item counts. + // Stage names appear in insertion order the first time they're seen. + std::vector stage_order; + std::unordered_map> stage_durations; + std::unordered_map> stage_item_counts; + + // Per-substage accumulated durations, keyed as "stage/substage". + std::unordered_map> substage_durations; + + // Total per-read duration. + std::vector read_durations; + + size_t slow_read_count = 0; + }; + + std::vector per_thread; + double slow_threshold_s; + + // Compute percentile p in [0,1] from a sorted vector. + static double percentile(const std::vector& sorted, double p); +}; + +} // namespace vg + +#endif // VG_GIRAFFE_STATS_HPP_INCLUDED diff --git a/src/minimizer_mapper.cpp b/src/minimizer_mapper.cpp index c678d3ecdc..353f823bd8 100644 --- a/src/minimizer_mapper.cpp +++ b/src/minimizer_mapper.cpp @@ -4,6 +4,7 @@ */ #include "minimizer_mapper.hpp" +#include "giraffe_stats.hpp" #include "crash.hpp" #include "annotation.hpp" @@ -634,11 +635,16 @@ vector MinimizerMapper::map_from_extensions(Alignment& aln) { // Cluster the seeds. Get sets of input seed indexes that go together. if (track_provenance) { funnel.stage("cluster"); + funnel.substage("cluster_seeds"); } // Find the clusters std::vector clusters = clusterer.cluster_seeds(seeds, get_distance_limit(aln.sequence().size())); - + + if (track_provenance) { + funnel.substage_stop(); + } + #ifdef debug_validate_clusters vector> all_clusters; all_clusters.emplace_back(clusters); @@ -1224,10 +1230,15 @@ vector MinimizerMapper::map_from_extensions(Alignment& aln) { // Stop this alignment funnel.stop(); - + + // Record aggregate stats and log slow reads if instrumentation is enabled. + if (giraffe_stats) { + giraffe_stats->record_read(funnel, aln.name()); + } + // Annotate with whatever's in the funnel funnel.annotate_mapped_alignment(mappings[0], track_correctness); - + if (track_provenance) { if (track_correctness) { annotate_with_minimizer_statistics(mappings[0], minimizers, seeds, seeds.size(), 0, funnel); diff --git a/src/minimizer_mapper.hpp b/src/minimizer_mapper.hpp index 41a376069e..cfa8aee47c 100644 --- a/src/minimizer_mapper.hpp +++ b/src/minimizer_mapper.hpp @@ -31,6 +31,9 @@ namespace vg { using namespace std; using namespace vg::io; +// Forward-declare GiraffeStats so the mapper can hold a pointer without a full include. +class GiraffeStats; + class MinimizerMapper : public AlignerClient { public: // Definitions used with minimizer indexes. @@ -432,6 +435,10 @@ class MinimizerMapper : public AlignerClient { static constexpr bool default_show_work = false; bool show_work = default_show_work; + /// If set, collect per-stage aggregate timing statistics and log slow reads. + /// Not owned by this object; caller manages lifetime. + class GiraffeStats* giraffe_stats = nullptr; + ////How many stdevs from fragment length distr mean do we cluster together? static constexpr double default_paired_distance_stdevs = 2.0; double paired_distance_stdevs = default_paired_distance_stdevs; diff --git a/src/minimizer_mapper_from_chains.cpp b/src/minimizer_mapper_from_chains.cpp index 26c86dce02..ccb0386b98 100644 --- a/src/minimizer_mapper_from_chains.cpp +++ b/src/minimizer_mapper_from_chains.cpp @@ -5,6 +5,7 @@ */ #include "minimizer_mapper.hpp" +#include "giraffe_stats.hpp" #include "annotation.hpp" #include "banded_global_aligner.hpp" @@ -718,6 +719,7 @@ vector MinimizerMapper::map_from_chains(Alignment& aln) { if (this->track_provenance) { funnel.stage("tree"); + funnel.substage("fill_in_forest"); } // Make them into a zip code tree @@ -725,6 +727,10 @@ vector MinimizerMapper::map_from_chains(Alignment& aln) { crash_unless(distance_index); zip_code_forest.fill_in_forest(seeds, *distance_index, aln.sequence().size() * zipcode_tree_scale); + if (this->track_provenance) { + funnel.substage_stop(); + } + #ifdef debug_print_forest if (show_work) { #pragma omp critical (cerr) @@ -1004,9 +1010,14 @@ vector MinimizerMapper::map_from_chains(Alignment& aln) { // Stop this alignment funnel.stop(); + // Record aggregate stats and log slow reads if instrumentation is enabled. + if (giraffe_stats) { + giraffe_stats->record_read(funnel, aln.name()); + } + // Annotate with whatever's in the funnel funnel.annotate_mapped_alignment(mappings[0], track_correctness); - + if (track_provenance) { if (track_correctness) { annotate_with_minimizer_statistics(mappings[0], minimizers, seeds, seeds.size(), chains.size(), funnel); @@ -1095,6 +1106,9 @@ void MinimizerMapper::do_chaining_on_trees(Alignment& aln, const ZipCodeForest& bool do_gapless_extension = aln.sequence().size() <= gapless_extension_limit; // First score all the zip code trees in the forest by summing the scores of their involved minimizers. + if (track_provenance) { + funnel.substage("score_trees"); + } vector tree_scores; double best_tree_score = 0; double second_best_tree_score = 0; @@ -1151,6 +1165,7 @@ void MinimizerMapper::do_chaining_on_trees(Alignment& aln, const ZipCodeForest& if (track_provenance) { + funnel.substage_stop(); funnel.stage("chain"); funnel.substage("chain"); } diff --git a/src/subcommand/giraffe_main.cpp b/src/subcommand/giraffe_main.cpp index f23fead27f..b419f91054 100644 --- a/src/subcommand/giraffe_main.cpp +++ b/src/subcommand/giraffe_main.cpp @@ -29,6 +29,7 @@ #include "../index_registry.hpp" #include "../utility.hpp" #include "../watchdog.hpp" +#include "../giraffe_stats.hpp" #include "../crash.hpp" #include @@ -765,6 +766,7 @@ int main_giraffe(int argc, char** argv) { constexpr int OPT_HAPLOTYPE_SAMPLING = 1104; constexpr int OPT_NUM_HAPLOTYPES = 1105; constexpr int OPT_NO_DIPLOID_SAMPLING = 1106; + constexpr int OPT_SLOW_READ_THRESHOLD = 1200; // initialize parameters with their default options @@ -832,7 +834,11 @@ int main_giraffe(int argc, char** argv) { bool track_position = MinimizerMapper::default_track_position; // Should we log our mapping decision making? bool show_work = MinimizerMapper::default_show_work; - + + // Reads longer than this (in seconds) get a per-stage breakdown logged to stderr. + // 0 disables per-read slow logging. The aggregate summary is always printed when > 0. + double slow_read_threshold = 0.0; + // Should we throw out our alignments instead of outputting them? bool discard_alignments = false; @@ -1109,6 +1115,7 @@ int main_giraffe(int argc, char** argv) { {"track-correctness", no_argument, 0, OPT_TRACK_CORRECTNESS}, {"track-position", no_argument, 0, OPT_TRACK_POSITION}, {"show-work", no_argument, 0, OPT_SHOW_WORK}, + {"slow-read-threshold", required_argument, 0, OPT_SLOW_READ_THRESHOLD}, {"threads", required_argument, 0, 't'}, }; parser->make_long_options(long_options); @@ -1362,7 +1369,11 @@ int main_giraffe(int argc, char** argv) { // Also turn on saving explanations Explainer::save_explanations = true; break; - + + case OPT_SLOW_READ_THRESHOLD: + slow_read_threshold = parse(optarg); + break; + case 't': set_thread_count(logger, optarg); break; @@ -1965,6 +1976,13 @@ int main_giraffe(int argc, char** argv) { // Work out the number of threads we will have size_t thread_count = omp_get_max_threads(); + // Set up per-stage timing instrumentation if requested. + unique_ptr giraffe_stats; + if (slow_read_threshold > 0.0) { + giraffe_stats.reset(new GiraffeStats(thread_count, slow_read_threshold)); + minimizer_mapper.giraffe_stats = giraffe_stats.get(); + } + // Set up counters per-thread for total reads mapped vector reads_mapped_by_thread(thread_count, 0); @@ -2415,8 +2433,11 @@ int main_giraffe(int argc, char** argv) { logger.info() << "Memory footprint: " << gbwt::inGigabytes(gbwt::memoryUsage()) << " GB" << endl; } - - + + if (giraffe_stats) { + giraffe_stats->print_summary(cerr); + } + if (report) { // Log output filename and mapping speed in reads/second/thread to report TSV report << output_filename << "\t" << reads_per_second_per_thread << endl; From 528ec4e5f306f8ab543056bc96ffb55a5bd43352 Mon Sep 17 00:00:00 2001 From: Zia <194475824+electricEpilith@users.noreply.github.com> Date: Fri, 3 Apr 2026 14:50:41 -0700 Subject: [PATCH 60/77] fix abs() errors on Mac Co-Authored-By: GitHub Copilot --- src/multipath_alignment_graph.cpp | 8 ++++---- src/subcommand/gampcompare_main.cpp | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/multipath_alignment_graph.cpp b/src/multipath_alignment_graph.cpp index 6cc2620f91..9038dff638 100644 --- a/src/multipath_alignment_graph.cpp +++ b/src/multipath_alignment_graph.cpp @@ -6505,7 +6505,7 @@ void MultipathAlignmentGraph::align(const Alignment& alignment, const HandleGrap for (const auto& path_node : path_nodes) { for (const auto& edge : path_node.edges) { const auto& next_node = path_nodes[edge.first]; - shift = max(shift, abs((next_node.begin - path_node.end) - edge.second)); + shift = max(shift, std::abs(static_cast((next_node.begin - path_node.end) - edge.second))); } } return shift; @@ -6529,7 +6529,7 @@ void MultipathAlignmentGraph::align(const Alignment& alignment, const HandleGrap ++in_degree[edge.first]; const auto& next_node = path_nodes[edge.first]; - size_t shift = abs((next_node.begin - path_node.end) - edge.second); + size_t shift = std::abs(static_cast((next_node.begin - path_node.end) - edge.second)); #ifdef debug_shift_pruning cerr << "shift DP reverse " << i << " <- " << edge.first << " with shift " << shift << " for total " << min_shift_rev[edge.first] + shift << endl; @@ -6555,7 +6555,7 @@ void MultipathAlignmentGraph::align(const Alignment& alignment, const HandleGrap else { for (auto& edge : path_node.edges) { const auto& next_node = path_nodes[edge.first]; - size_t shift = abs((next_node.begin - path_node.end) - edge.second); + size_t shift = std::abs(static_cast((next_node.begin - path_node.end) - edge.second)); #ifdef debug_shift_pruning cerr << "shift DP forward " << i << " -> " << edge.first << " with shift " << shift << " for total " << min_shift_fwd[i] + shift << endl; #endif @@ -6577,7 +6577,7 @@ void MultipathAlignmentGraph::align(const Alignment& alignment, const HandleGrap for (size_t j = 0; j < path_node.edges.size(); ++j) { auto& edge = path_node.edges[j]; const auto& next_node = path_nodes[edge.first]; - size_t shift = abs((next_node.begin - path_node.end) - edge.second); + size_t shift = std::abs(static_cast((next_node.begin - path_node.end) - edge.second)); size_t min_edge_shift = min_shift_fwd[i] + shift + min_shift_rev[edge.first]; diff --git a/src/subcommand/gampcompare_main.cpp b/src/subcommand/gampcompare_main.cpp index 72c84c289e..6730fc8d81 100644 --- a/src/subcommand/gampcompare_main.cpp +++ b/src/subcommand/gampcompare_main.cpp @@ -216,7 +216,7 @@ int main_gampcompare(int argc, char** argv) { if (path_true_positions[i].second == path_mapped_positions[j].second) { // there is a pair of positions on the same strand of the same path abs_dist = min(abs_dist, - abs(path_true_positions[i].first - path_mapped_positions[j].first)); + std::abs(static_cast(path_true_positions[i].first - path_mapped_positions[j].first))); } } } From a3760d19cd29a1fff0624355902f73adb503cd59 Mon Sep 17 00:00:00 2001 From: Zia <194475824+electricEpilith@users.noreply.github.com> Date: Fri, 3 Apr 2026 19:31:43 -0700 Subject: [PATCH 61/77] additional abs() fix Co-Authored-By: GitHub Copilot --- src/multipath_mapper.cpp | 2 +- src/recombinator.cpp | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/multipath_mapper.cpp b/src/multipath_mapper.cpp index 4632edee8b..4241fa12e6 100644 --- a/src/multipath_mapper.cpp +++ b/src/multipath_mapper.cpp @@ -2448,7 +2448,7 @@ namespace vg { // in the left_idxs and right_idxs vectors int64_t target_len = 2 * seq_len - left_side.clip_length - right_side.clip_length; auto distance_diff = [&](size_t l, size_t r) { - return abs(get<2>(left_sites[left_idxs[l]]) + get<2>(right_sites[right_idxs[r]]) - target_len); + return std::abs(static_cast(get<2>(left_sites[left_idxs[l]]) + get<2>(right_sites[right_idxs[r]]) - target_len)); }; // sweep to identify pairs that most nearly align diff --git a/src/recombinator.cpp b/src/recombinator.cpp index a9aaed4b10..07915118ed 100644 --- a/src/recombinator.cpp +++ b/src/recombinator.cpp @@ -1585,7 +1585,7 @@ void add_path(const gbwt::GBWT& source, gbwt::size_type path_id, gbwt::GBWTBuild gbwt::PathName path_name = source.metadata.path(path_id); std::string sample_name = source.metadata.sample(path_name.sample); std::string contig_name = source.metadata.contig(path_name.contig); - if (sample_name == gbwtgraph::REFERENCE_PATH_SAMPLE_NAME) { + if (sample_name == gbwtgraph::GENERIC_PATH_SAMPLE_NAME) { metadata.add_generic_path(contig_name); } else { // Reference samples will be copied later. From 7920c76d12b008882a72149e8b72ec6fcdec48ca Mon Sep 17 00:00:00 2001 From: Zia <194475824+electricEpilith@users.noreply.github.com> Date: Fri, 3 Apr 2026 20:07:36 -0700 Subject: [PATCH 62/77] minor print changes Co-Authored-By: GitHub Copilot --- src/giraffe_stats.cpp | 15 ++++++--------- src/giraffe_stats.hpp | 33 ++++++++++++++++++++++++++++++--- 2 files changed, 36 insertions(+), 12 deletions(-) diff --git a/src/giraffe_stats.cpp b/src/giraffe_stats.cpp index 3e0b5129bc..88980960bb 100644 --- a/src/giraffe_stats.cpp +++ b/src/giraffe_stats.cpp @@ -6,6 +6,7 @@ #include #include +#include #include #include #include @@ -31,9 +32,7 @@ void GiraffeStats::record_read(const Funnel& funnel, const std::string& read_nam // Accumulate per-stage data and optionally build the slow-read message. std::ostringstream slow_msg; if (is_slow) { - slow_msg << "warning[vg::Giraffe]: Slow read \"" << read_name - << "\" took " << std::fixed << std::setprecision(3) - << total << "s:\n"; + slow_msg << std::format("warning[vg::Giraffe]: Slow read \"{}\" took {:.3f}s:\n", read_name, total); } funnel.for_each_stage([&](const std::string& stage, @@ -55,9 +54,7 @@ void GiraffeStats::record_read(const Funnel& funnel, const std::string& read_nam } if (is_slow) { - slow_msg << " " << stage << ": " - << std::fixed << std::setprecision(3) << duration << "s" - << " (" << result_sizes.size() << " items)\n"; + slow_msg << std::format(" {}: {:.3f}s ({} items)\n", stage, duration, result_sizes.size()); for (auto& kv : sub_durations) { slow_msg << " " << kv.first << ": " << std::fixed << std::setprecision(3) << kv.second << "s\n"; @@ -87,9 +84,9 @@ void GiraffeStats::print_summary(std::ostream& out) const { // Use the stage order from thread 0 (or whichever thread saw stages first), // then append any stages seen by other threads. std::vector stage_order; - std::unordered_map> stage_durations; - std::unordered_map> stage_item_counts; - std::unordered_map> substage_durations; + std::unordered_map, StringHash, StringEqual> stage_durations; + std::unordered_map, StringHash, StringEqual> stage_item_counts; + std::unordered_map, StringHash, StringEqual> substage_durations; std::vector read_durations; size_t slow_read_count = 0; size_t total_reads = 0; diff --git a/src/giraffe_stats.hpp b/src/giraffe_stats.hpp index bb3316ed97..125ebdd8b2 100644 --- a/src/giraffe_stats.hpp +++ b/src/giraffe_stats.hpp @@ -17,6 +17,33 @@ namespace vg { using namespace std; +/** + * Transparent hasher for std::string keys in unordered_map. + * Allows heterogeneous lookup with string_view and other compatible types. + */ +struct StringHash { + using is_transparent = void; + size_t operator()(std::string_view str) const { + return std::hash()(str); + } + size_t operator()(const std::string& str) const { + return std::hash()(str); + } +}; + +/** + * Transparent equality for std::string keys in unordered_map. + */ +struct StringEqual { + using is_transparent = void; + bool operator()(std::string_view lhs, std::string_view rhs) const { + return lhs == rhs; + } + bool operator()(const std::string& lhs, const std::string& rhs) const { + return lhs == rhs; + } +}; + /** * Thread-safe aggregate statistics collector for the Giraffe mapper. * @@ -49,11 +76,11 @@ class GiraffeStats { // Per-stage accumulated durations and item counts. // Stage names appear in insertion order the first time they're seen. std::vector stage_order; - std::unordered_map> stage_durations; - std::unordered_map> stage_item_counts; + std::unordered_map, StringHash, StringEqual> stage_durations; + std::unordered_map, StringHash, StringEqual> stage_item_counts; // Per-substage accumulated durations, keyed as "stage/substage". - std::unordered_map> substage_durations; + std::unordered_map, StringHash, StringEqual> substage_durations; // Total per-read duration. std::vector read_durations; From 190469a6f4eb57e1e9fe24850feb634c2cbc72b0 Mon Sep 17 00:00:00 2001 From: Zia Truong <194475824+electricEpilith@users.noreply.github.com> Date: Fri, 3 Apr 2026 22:39:15 -0700 Subject: [PATCH 63/77] Revert "minor print changes" This reverts commit 7920c76d12b008882a72149e8b72ec6fcdec48ca. --- src/giraffe_stats.cpp | 15 +++++++++------ src/giraffe_stats.hpp | 33 +++------------------------------ 2 files changed, 12 insertions(+), 36 deletions(-) diff --git a/src/giraffe_stats.cpp b/src/giraffe_stats.cpp index 88980960bb..3e0b5129bc 100644 --- a/src/giraffe_stats.cpp +++ b/src/giraffe_stats.cpp @@ -6,7 +6,6 @@ #include #include -#include #include #include #include @@ -32,7 +31,9 @@ void GiraffeStats::record_read(const Funnel& funnel, const std::string& read_nam // Accumulate per-stage data and optionally build the slow-read message. std::ostringstream slow_msg; if (is_slow) { - slow_msg << std::format("warning[vg::Giraffe]: Slow read \"{}\" took {:.3f}s:\n", read_name, total); + slow_msg << "warning[vg::Giraffe]: Slow read \"" << read_name + << "\" took " << std::fixed << std::setprecision(3) + << total << "s:\n"; } funnel.for_each_stage([&](const std::string& stage, @@ -54,7 +55,9 @@ void GiraffeStats::record_read(const Funnel& funnel, const std::string& read_nam } if (is_slow) { - slow_msg << std::format(" {}: {:.3f}s ({} items)\n", stage, duration, result_sizes.size()); + slow_msg << " " << stage << ": " + << std::fixed << std::setprecision(3) << duration << "s" + << " (" << result_sizes.size() << " items)\n"; for (auto& kv : sub_durations) { slow_msg << " " << kv.first << ": " << std::fixed << std::setprecision(3) << kv.second << "s\n"; @@ -84,9 +87,9 @@ void GiraffeStats::print_summary(std::ostream& out) const { // Use the stage order from thread 0 (or whichever thread saw stages first), // then append any stages seen by other threads. std::vector stage_order; - std::unordered_map, StringHash, StringEqual> stage_durations; - std::unordered_map, StringHash, StringEqual> stage_item_counts; - std::unordered_map, StringHash, StringEqual> substage_durations; + std::unordered_map> stage_durations; + std::unordered_map> stage_item_counts; + std::unordered_map> substage_durations; std::vector read_durations; size_t slow_read_count = 0; size_t total_reads = 0; diff --git a/src/giraffe_stats.hpp b/src/giraffe_stats.hpp index 125ebdd8b2..bb3316ed97 100644 --- a/src/giraffe_stats.hpp +++ b/src/giraffe_stats.hpp @@ -17,33 +17,6 @@ namespace vg { using namespace std; -/** - * Transparent hasher for std::string keys in unordered_map. - * Allows heterogeneous lookup with string_view and other compatible types. - */ -struct StringHash { - using is_transparent = void; - size_t operator()(std::string_view str) const { - return std::hash()(str); - } - size_t operator()(const std::string& str) const { - return std::hash()(str); - } -}; - -/** - * Transparent equality for std::string keys in unordered_map. - */ -struct StringEqual { - using is_transparent = void; - bool operator()(std::string_view lhs, std::string_view rhs) const { - return lhs == rhs; - } - bool operator()(const std::string& lhs, const std::string& rhs) const { - return lhs == rhs; - } -}; - /** * Thread-safe aggregate statistics collector for the Giraffe mapper. * @@ -76,11 +49,11 @@ class GiraffeStats { // Per-stage accumulated durations and item counts. // Stage names appear in insertion order the first time they're seen. std::vector stage_order; - std::unordered_map, StringHash, StringEqual> stage_durations; - std::unordered_map, StringHash, StringEqual> stage_item_counts; + std::unordered_map> stage_durations; + std::unordered_map> stage_item_counts; // Per-substage accumulated durations, keyed as "stage/substage". - std::unordered_map, StringHash, StringEqual> substage_durations; + std::unordered_map> substage_durations; // Total per-read duration. std::vector read_durations; From 3e573bd54129801c6f7f1ff31c697e815500f887 Mon Sep 17 00:00:00 2001 From: Zia Truong <194475824+electricEpilith@users.noreply.github.com> Date: Fri, 3 Apr 2026 22:40:54 -0700 Subject: [PATCH 64/77] Revert "add (a substantial amount of) instrumentation for vg giraffe" This reverts commit b81e331a8d4d7b19286530e521614e587e17ec73. --- src/funnel.cpp | 4 - src/funnel.hpp | 4 - src/giraffe_stats.cpp | 200 --------------------------- src/giraffe_stats.hpp | 73 ---------- src/minimizer_mapper.cpp | 17 +-- src/minimizer_mapper.hpp | 7 - src/minimizer_mapper_from_chains.cpp | 17 +-- src/subcommand/giraffe_main.cpp | 29 +--- 8 files changed, 8 insertions(+), 343 deletions(-) delete mode 100644 src/giraffe_stats.cpp delete mode 100644 src/giraffe_stats.hpp diff --git a/src/funnel.cpp b/src/funnel.cpp index 06cec8fe9d..ebf796b3aa 100644 --- a/src/funnel.cpp +++ b/src/funnel.cpp @@ -428,10 +428,6 @@ size_t Funnel::latest() const { return stages.back().items.size() - 1; } -double Funnel::total_seconds() const { - return chrono::duration_cast>(stop_time - start_time).count(); -} - void Funnel::for_each_stage(const function&, const vector&, const vector&, const double&, const std::unordered_map&)>& callback) const { for (auto& stage : stages) { // Make a vector of item sizes diff --git a/src/funnel.hpp b/src/funnel.hpp index 7a08d524d2..7c0add6a4d 100644 --- a/src/funnel.hpp +++ b/src/funnel.hpp @@ -223,10 +223,6 @@ class Funnel { /// Get the index of the most recent item created in the current stage. size_t latest() const; - - /// Get the total elapsed seconds from start() to stop(). - /// Only valid after stop() has been called. - double total_seconds() const; /// Call the given callback with stage name, a vector of result item sizes /// at that stage, a vector of correct item scores at that stage (if any), diff --git a/src/giraffe_stats.cpp b/src/giraffe_stats.cpp deleted file mode 100644 index 3e0b5129bc..0000000000 --- a/src/giraffe_stats.cpp +++ /dev/null @@ -1,200 +0,0 @@ -/** - * \file giraffe_stats.cpp - */ - -#include "giraffe_stats.hpp" - -#include -#include -#include -#include -#include -#include - -namespace vg { - -GiraffeStats::GiraffeStats(size_t thread_count, double slow_threshold_s) - : per_thread(thread_count), slow_threshold_s(slow_threshold_s) {} - -void GiraffeStats::record_read(const Funnel& funnel, const std::string& read_name) { - int tid = omp_get_thread_num(); - ThreadData& td = per_thread.at(tid); - - double total = funnel.total_seconds(); - td.read_durations.push_back(total); - - bool is_slow = slow_threshold_s > 0.0 && total > slow_threshold_s; - if (is_slow) { - td.slow_read_count++; - } - - // Accumulate per-stage data and optionally build the slow-read message. - std::ostringstream slow_msg; - if (is_slow) { - slow_msg << "warning[vg::Giraffe]: Slow read \"" << read_name - << "\" took " << std::fixed << std::setprecision(3) - << total << "s:\n"; - } - - funnel.for_each_stage([&](const std::string& stage, - const std::vector& result_sizes, - const std::vector& /*correct_scores*/, - const std::vector& /*noncorrect_scores*/, - const double& duration, - const std::unordered_map& sub_durations) { - // Record into per-thread storage. - if (!td.stage_durations.count(stage)) { - td.stage_order.push_back(stage); - } - td.stage_durations[stage].push_back(duration); - td.stage_item_counts[stage].push_back(result_sizes.size()); - - for (auto& kv : sub_durations) { - std::string key = stage + "/" + kv.first; - td.substage_durations[key].push_back(kv.second); - } - - if (is_slow) { - slow_msg << " " << stage << ": " - << std::fixed << std::setprecision(3) << duration << "s" - << " (" << result_sizes.size() << " items)\n"; - for (auto& kv : sub_durations) { - slow_msg << " " << kv.first << ": " - << std::fixed << std::setprecision(3) << kv.second << "s\n"; - } - } - }); - - if (is_slow) { - #pragma omp critical (cerr) - std::cerr << slow_msg.str() << std::flush; - } -} - -double GiraffeStats::percentile(const std::vector& sorted, double p) { - if (sorted.empty()) return 0.0; - if (sorted.size() == 1) return sorted[0]; - double idx = p * (sorted.size() - 1); - size_t lo = (size_t)idx; - size_t hi = lo + 1; - if (hi >= sorted.size()) return sorted.back(); - double frac = idx - lo; - return sorted[lo] * (1.0 - frac) + sorted[hi] * frac; -} - -void GiraffeStats::print_summary(std::ostream& out) const { - // Merge all thread data. - // Use the stage order from thread 0 (or whichever thread saw stages first), - // then append any stages seen by other threads. - std::vector stage_order; - std::unordered_map> stage_durations; - std::unordered_map> stage_item_counts; - std::unordered_map> substage_durations; - std::vector read_durations; - size_t slow_read_count = 0; - size_t total_reads = 0; - - for (auto& td : per_thread) { - // Merge stage order (preserve first-seen ordering). - for (auto& s : td.stage_order) { - if (!stage_durations.count(s)) { - stage_order.push_back(s); - } - } - for (auto& kv : td.stage_durations) { - auto& dst = stage_durations[kv.first]; - dst.insert(dst.end(), kv.second.begin(), kv.second.end()); - } - for (auto& kv : td.stage_item_counts) { - auto& dst = stage_item_counts[kv.first]; - dst.insert(dst.end(), kv.second.begin(), kv.second.end()); - } - for (auto& kv : td.substage_durations) { - auto& dst = substage_durations[kv.first]; - dst.insert(dst.end(), kv.second.begin(), kv.second.end()); - } - read_durations.insert(read_durations.end(), - td.read_durations.begin(), td.read_durations.end()); - slow_read_count += td.slow_read_count; - total_reads += td.read_durations.size(); - } - - if (total_reads == 0) return; - - // Sort per-read durations for percentiles. - std::vector sorted_reads = read_durations; - std::sort(sorted_reads.begin(), sorted_reads.end()); - - // Print header. - out << "\n=== Giraffe Per-Stage Timing (" << total_reads << " reads) ===\n"; - out << std::left - << std::setw(28) << "Stage" - << std::right - << std::setw(10) << "Mean(ms)" - << std::setw(10) << "P50(ms)" - << std::setw(10) << "P95(ms)" - << std::setw(10) << "P99(ms)" - << std::setw(10) << "Max(ms)" - << std::setw(12) << "MeanItems" - << std::setw(10) << "MaxItems" - << "\n"; - - // Helper to print one row. - auto print_row = [&](const std::string& label, std::vector& durs, - const std::vector* items) { - std::sort(durs.begin(), durs.end()); - double mean_ms = 1000.0 * std::accumulate(durs.begin(), durs.end(), 0.0) / durs.size(); - double p50_ms = 1000.0 * percentile(durs, 0.50); - double p95_ms = 1000.0 * percentile(durs, 0.95); - double p99_ms = 1000.0 * percentile(durs, 0.99); - double max_ms = 1000.0 * durs.back(); - - out << std::left << std::setw(28) << label << std::right - << std::fixed << std::setprecision(2) - << std::setw(10) << mean_ms - << std::setw(10) << p50_ms - << std::setw(10) << p95_ms - << std::setw(10) << p99_ms - << std::setw(10) << max_ms; - - if (items && !items->empty()) { - double mean_items = (double)std::accumulate(items->begin(), items->end(), (size_t)0) / items->size(); - size_t max_items = *std::max_element(items->begin(), items->end()); - out << std::setw(12) << std::setprecision(1) << mean_items - << std::setw(10) << max_items; - } - out << "\n"; - }; - - for (auto& stage : stage_order) { - print_row(stage, stage_durations[stage], &stage_item_counts[stage]); - - // Print any substages for this stage, indented. - for (auto& kv : substage_durations) { - // Key format is "stage/substage". - size_t slash = kv.first.find('/'); - if (slash == std::string::npos) continue; - if (kv.first.substr(0, slash) != stage) continue; - std::string substage_label = " " + kv.first.substr(slash + 1); - print_row(substage_label, kv.second, nullptr); - } - } - - // Total read timing. - double mean_ms = 1000.0 * std::accumulate(read_durations.begin(), read_durations.end(), 0.0) / total_reads; - double p99_ms = 1000.0 * percentile(sorted_reads, 0.99); - double max_ms = 1000.0 * sorted_reads.back(); - - out << "\nTotal per-read: mean=" << std::fixed << std::setprecision(2) << mean_ms - << "ms, P99=" << p99_ms << "ms, max=" << max_ms << "ms\n"; - - if (slow_threshold_s > 0.0) { - out << "Slow reads (>" << slow_threshold_s << "s): " - << slow_read_count << " / " << total_reads - << " (" << std::fixed << std::setprecision(3) - << 100.0 * slow_read_count / total_reads << "%)\n"; - } - out << std::flush; -} - -} // namespace vg diff --git a/src/giraffe_stats.hpp b/src/giraffe_stats.hpp deleted file mode 100644 index bb3316ed97..0000000000 --- a/src/giraffe_stats.hpp +++ /dev/null @@ -1,73 +0,0 @@ -#ifndef VG_GIRAFFE_STATS_HPP_INCLUDED -#define VG_GIRAFFE_STATS_HPP_INCLUDED - -/** - * \file giraffe_stats.hpp - * Aggregate per-stage timing statistics across reads for the Giraffe mapper. - */ - -#include "funnel.hpp" - -#include -#include -#include -#include - -namespace vg { - -using namespace std; - -/** - * Thread-safe aggregate statistics collector for the Giraffe mapper. - * - * Each thread pushes data into its own ThreadData slot (no locking on the hot - * path). Summary statistics are computed and printed after all reads are - * mapped. - * - * Also logs a per-stage breakdown to stderr for any individual read whose - * total mapping time exceeds a configurable threshold. - */ -class GiraffeStats { -public: - GiraffeStats(size_t thread_count, double slow_threshold_s); - - /** - * Record timing/item data from one read's Funnel. - * Must be called after funnel.stop(). - * Uses omp_get_thread_num() to select the per-thread slot. - */ - void record_read(const Funnel& funnel, const std::string& read_name); - - /** - * Print a summary table of per-stage timing percentiles to out. - * Thread-safe to call after all record_read() calls are done. - */ - void print_summary(std::ostream& out) const; - -private: - struct ThreadData { - // Per-stage accumulated durations and item counts. - // Stage names appear in insertion order the first time they're seen. - std::vector stage_order; - std::unordered_map> stage_durations; - std::unordered_map> stage_item_counts; - - // Per-substage accumulated durations, keyed as "stage/substage". - std::unordered_map> substage_durations; - - // Total per-read duration. - std::vector read_durations; - - size_t slow_read_count = 0; - }; - - std::vector per_thread; - double slow_threshold_s; - - // Compute percentile p in [0,1] from a sorted vector. - static double percentile(const std::vector& sorted, double p); -}; - -} // namespace vg - -#endif // VG_GIRAFFE_STATS_HPP_INCLUDED diff --git a/src/minimizer_mapper.cpp b/src/minimizer_mapper.cpp index 353f823bd8..c678d3ecdc 100644 --- a/src/minimizer_mapper.cpp +++ b/src/minimizer_mapper.cpp @@ -4,7 +4,6 @@ */ #include "minimizer_mapper.hpp" -#include "giraffe_stats.hpp" #include "crash.hpp" #include "annotation.hpp" @@ -635,16 +634,11 @@ vector MinimizerMapper::map_from_extensions(Alignment& aln) { // Cluster the seeds. Get sets of input seed indexes that go together. if (track_provenance) { funnel.stage("cluster"); - funnel.substage("cluster_seeds"); } // Find the clusters std::vector clusters = clusterer.cluster_seeds(seeds, get_distance_limit(aln.sequence().size())); - - if (track_provenance) { - funnel.substage_stop(); - } - + #ifdef debug_validate_clusters vector> all_clusters; all_clusters.emplace_back(clusters); @@ -1230,15 +1224,10 @@ vector MinimizerMapper::map_from_extensions(Alignment& aln) { // Stop this alignment funnel.stop(); - - // Record aggregate stats and log slow reads if instrumentation is enabled. - if (giraffe_stats) { - giraffe_stats->record_read(funnel, aln.name()); - } - + // Annotate with whatever's in the funnel funnel.annotate_mapped_alignment(mappings[0], track_correctness); - + if (track_provenance) { if (track_correctness) { annotate_with_minimizer_statistics(mappings[0], minimizers, seeds, seeds.size(), 0, funnel); diff --git a/src/minimizer_mapper.hpp b/src/minimizer_mapper.hpp index cfa8aee47c..41a376069e 100644 --- a/src/minimizer_mapper.hpp +++ b/src/minimizer_mapper.hpp @@ -31,9 +31,6 @@ namespace vg { using namespace std; using namespace vg::io; -// Forward-declare GiraffeStats so the mapper can hold a pointer without a full include. -class GiraffeStats; - class MinimizerMapper : public AlignerClient { public: // Definitions used with minimizer indexes. @@ -435,10 +432,6 @@ class MinimizerMapper : public AlignerClient { static constexpr bool default_show_work = false; bool show_work = default_show_work; - /// If set, collect per-stage aggregate timing statistics and log slow reads. - /// Not owned by this object; caller manages lifetime. - class GiraffeStats* giraffe_stats = nullptr; - ////How many stdevs from fragment length distr mean do we cluster together? static constexpr double default_paired_distance_stdevs = 2.0; double paired_distance_stdevs = default_paired_distance_stdevs; diff --git a/src/minimizer_mapper_from_chains.cpp b/src/minimizer_mapper_from_chains.cpp index ccb0386b98..26c86dce02 100644 --- a/src/minimizer_mapper_from_chains.cpp +++ b/src/minimizer_mapper_from_chains.cpp @@ -5,7 +5,6 @@ */ #include "minimizer_mapper.hpp" -#include "giraffe_stats.hpp" #include "annotation.hpp" #include "banded_global_aligner.hpp" @@ -719,7 +718,6 @@ vector MinimizerMapper::map_from_chains(Alignment& aln) { if (this->track_provenance) { funnel.stage("tree"); - funnel.substage("fill_in_forest"); } // Make them into a zip code tree @@ -727,10 +725,6 @@ vector MinimizerMapper::map_from_chains(Alignment& aln) { crash_unless(distance_index); zip_code_forest.fill_in_forest(seeds, *distance_index, aln.sequence().size() * zipcode_tree_scale); - if (this->track_provenance) { - funnel.substage_stop(); - } - #ifdef debug_print_forest if (show_work) { #pragma omp critical (cerr) @@ -1010,14 +1004,9 @@ vector MinimizerMapper::map_from_chains(Alignment& aln) { // Stop this alignment funnel.stop(); - // Record aggregate stats and log slow reads if instrumentation is enabled. - if (giraffe_stats) { - giraffe_stats->record_read(funnel, aln.name()); - } - // Annotate with whatever's in the funnel funnel.annotate_mapped_alignment(mappings[0], track_correctness); - + if (track_provenance) { if (track_correctness) { annotate_with_minimizer_statistics(mappings[0], minimizers, seeds, seeds.size(), chains.size(), funnel); @@ -1106,9 +1095,6 @@ void MinimizerMapper::do_chaining_on_trees(Alignment& aln, const ZipCodeForest& bool do_gapless_extension = aln.sequence().size() <= gapless_extension_limit; // First score all the zip code trees in the forest by summing the scores of their involved minimizers. - if (track_provenance) { - funnel.substage("score_trees"); - } vector tree_scores; double best_tree_score = 0; double second_best_tree_score = 0; @@ -1165,7 +1151,6 @@ void MinimizerMapper::do_chaining_on_trees(Alignment& aln, const ZipCodeForest& if (track_provenance) { - funnel.substage_stop(); funnel.stage("chain"); funnel.substage("chain"); } diff --git a/src/subcommand/giraffe_main.cpp b/src/subcommand/giraffe_main.cpp index b419f91054..f23fead27f 100644 --- a/src/subcommand/giraffe_main.cpp +++ b/src/subcommand/giraffe_main.cpp @@ -29,7 +29,6 @@ #include "../index_registry.hpp" #include "../utility.hpp" #include "../watchdog.hpp" -#include "../giraffe_stats.hpp" #include "../crash.hpp" #include @@ -766,7 +765,6 @@ int main_giraffe(int argc, char** argv) { constexpr int OPT_HAPLOTYPE_SAMPLING = 1104; constexpr int OPT_NUM_HAPLOTYPES = 1105; constexpr int OPT_NO_DIPLOID_SAMPLING = 1106; - constexpr int OPT_SLOW_READ_THRESHOLD = 1200; // initialize parameters with their default options @@ -834,11 +832,7 @@ int main_giraffe(int argc, char** argv) { bool track_position = MinimizerMapper::default_track_position; // Should we log our mapping decision making? bool show_work = MinimizerMapper::default_show_work; - - // Reads longer than this (in seconds) get a per-stage breakdown logged to stderr. - // 0 disables per-read slow logging. The aggregate summary is always printed when > 0. - double slow_read_threshold = 0.0; - + // Should we throw out our alignments instead of outputting them? bool discard_alignments = false; @@ -1115,7 +1109,6 @@ int main_giraffe(int argc, char** argv) { {"track-correctness", no_argument, 0, OPT_TRACK_CORRECTNESS}, {"track-position", no_argument, 0, OPT_TRACK_POSITION}, {"show-work", no_argument, 0, OPT_SHOW_WORK}, - {"slow-read-threshold", required_argument, 0, OPT_SLOW_READ_THRESHOLD}, {"threads", required_argument, 0, 't'}, }; parser->make_long_options(long_options); @@ -1369,11 +1362,7 @@ int main_giraffe(int argc, char** argv) { // Also turn on saving explanations Explainer::save_explanations = true; break; - - case OPT_SLOW_READ_THRESHOLD: - slow_read_threshold = parse(optarg); - break; - + case 't': set_thread_count(logger, optarg); break; @@ -1976,13 +1965,6 @@ int main_giraffe(int argc, char** argv) { // Work out the number of threads we will have size_t thread_count = omp_get_max_threads(); - // Set up per-stage timing instrumentation if requested. - unique_ptr giraffe_stats; - if (slow_read_threshold > 0.0) { - giraffe_stats.reset(new GiraffeStats(thread_count, slow_read_threshold)); - minimizer_mapper.giraffe_stats = giraffe_stats.get(); - } - // Set up counters per-thread for total reads mapped vector reads_mapped_by_thread(thread_count, 0); @@ -2433,11 +2415,8 @@ int main_giraffe(int argc, char** argv) { logger.info() << "Memory footprint: " << gbwt::inGigabytes(gbwt::memoryUsage()) << " GB" << endl; } - - if (giraffe_stats) { - giraffe_stats->print_summary(cerr); - } - + + if (report) { // Log output filename and mapping speed in reads/second/thread to report TSV report << output_filename << "\t" << reads_per_second_per_thread << endl; From 6cf266c9bf6ccc362489d88595563fd196a47613 Mon Sep 17 00:00:00 2001 From: Zia <194475824+electricEpilith@users.noreply.github.com> Date: Sat, 4 Apr 2026 00:18:04 -0700 Subject: [PATCH 65/77] second try at instrumentation initial plan by Claude Opus 4.6 Co-Authored-By: Claude Sonnet 4.6 Co-Authored-By: GitHub Copilot --- src/minimizer_mapper.cpp | 35 ++++++++++++++++++++-- src/minimizer_mapper.hpp | 45 ++++++++++++++++++++++++++-- src/minimizer_mapper_from_chains.cpp | 45 +++++++++++++++++++++++----- src/subcommand/giraffe_main.cpp | 26 ++++++++++++++++ 4 files changed, 140 insertions(+), 11 deletions(-) diff --git a/src/minimizer_mapper.cpp b/src/minimizer_mapper.cpp index c678d3ecdc..c1e5744cb0 100644 --- a/src/minimizer_mapper.cpp +++ b/src/minimizer_mapper.cpp @@ -30,9 +30,12 @@ #include #include +#include #include #include +#include + // Turn on debugging prints //#define debug // Turn on printing of minimizer fact tables @@ -621,17 +624,34 @@ vector MinimizerMapper::map_from_extensions(Alignment& aln) { return aln.sequence(); }); + // Set up per-stage timing. + stage_timings_t* per_thread_timings = nullptr; + if (!stage_timings_by_thread.empty()) { + per_thread_timings = &stage_timings_by_thread[omp_get_thread_num()]; + } + auto stage_clock_now = []() { return std::chrono::high_resolution_clock::now(); }; + auto stage_ns = [](std::chrono::high_resolution_clock::time_point s, + std::chrono::high_resolution_clock::time_point e) { + return std::chrono::duration(e - s).count(); + }; + auto total_start = stage_clock_now(); + // Minimizers sorted by position + auto minimizer_stage_start = stage_clock_now(); std::vector minimizers_in_read = this->find_minimizers(aln.sequence(), funnel); // Indexes of minimizers, sorted into score order, best score first std::vector minimizer_score_order = sort_minimizers_by_score(minimizers_in_read, rng); // Minimizers sorted by best score first VectorView minimizers{minimizers_in_read, minimizer_score_order}; + if (per_thread_timings) { per_thread_timings->minimizer_ns += stage_ns(minimizer_stage_start, stage_clock_now()); } // Find the seeds and mark the minimizers that were located. + auto seed_stage_start = stage_clock_now(); vector seeds = this->find_seeds(minimizers_in_read, minimizers, aln, funnel); - + if (per_thread_timings) { per_thread_timings->seed_ns += stage_ns(seed_stage_start, stage_clock_now()); } + // Cluster the seeds. Get sets of input seed indexes that go together. + auto tree_stage_start = stage_clock_now(); // "tree" slot holds cluster+extend for extensions path if (track_provenance) { funnel.stage("cluster"); } @@ -832,6 +852,8 @@ vector MinimizerMapper::map_from_extensions(Alignment& aln) { }); std::vector cluster_extension_scores = this->score_extensions(cluster_extensions, aln, funnel); + if (per_thread_timings) { per_thread_timings->chain_ns += stage_ns(tree_stage_start, stage_clock_now()); } + auto align_stage_start = stage_clock_now(); if (track_provenance) { funnel.stage("align"); } @@ -1073,11 +1095,13 @@ vector MinimizerMapper::map_from_extensions(Alignment& aln) { supplementaries = std::move(identify_supplementary_alignments(alignments, funnel)); } + if (per_thread_timings) { per_thread_timings->align_ns += stage_ns(align_stage_start, stage_clock_now()); } + auto winner_stage_start = stage_clock_now(); if (track_provenance) { // Now say we are finding the winner(s) funnel.stage("winner"); } - + // Fill this in with the alignments we will output as mappings vector mappings; mappings.reserve(min(alignments.size(), max_multimaps)); @@ -1274,6 +1298,13 @@ vector MinimizerMapper::map_from_extensions(Alignment& aln) { } } + // Record winner stage and total map_from_extensions time. + if (per_thread_timings) { + per_thread_timings->winner_ns += stage_ns(winner_stage_start, stage_clock_now()); + per_thread_timings->total_ns += stage_ns(total_start, stage_clock_now()); + per_thread_timings->read_count++; + } + return mappings; } diff --git a/src/minimizer_mapper.hpp b/src/minimizer_mapper.hpp index 41a376069e..e3dd804ba6 100644 --- a/src/minimizer_mapper.hpp +++ b/src/minimizer_mapper.hpp @@ -590,7 +590,45 @@ class MinimizerMapper : public AlignerClient { return this->occs.second; } }; - + + /// Struct for per-stage timing accumulators + struct stage_timings_t { + double minimizer_ns = 0; ///< find_minimizers + flag_repetitive + sort + double seed_ns = 0; ///< find_seeds + double tree_ns = 0; ///< zip code forest fill + to_anchors + tree scoring + double chain_ns = 0; ///< do_chaining_on_trees (incl. gapless extension) + double align_ns = 0; ///< do_alignment_on_chains + double winner_ns = 0; ///< pick_mappings + MAPQ + annotation + double total_ns = 0; ///< full map_from_chains/map_from_extensions wall time + size_t read_count = 0; + + inline stage_timings_t& operator+=(const stage_timings_t& other) { + minimizer_ns += other.minimizer_ns; + seed_ns += other.seed_ns; + tree_ns += other.tree_ns; + chain_ns += other.chain_ns; + align_ns += other.align_ns; + winner_ns += other.winner_ns; + total_ns += other.total_ns; + read_count += other.read_count; + return *this; + } + }; + + /// Initialize per-thread stage timing accumulators for the given number of threads + void initialize_stage_timings(size_t thread_count) { + stage_timings_by_thread.assign(thread_count, stage_timings_t{}); + } + + /// Aggregate stage timings across all threads. + stage_timings_t get_stage_timings() const { + stage_timings_t total; + for (auto& t : stage_timings_by_thread) { + total += t; + } + return total; + } + protected: /// Types of paired alignments in paired-end mapping @@ -784,7 +822,10 @@ class MinimizerMapper : public AlignerClient { */ double get_read_coverage(const Alignment& aln, const VectorView>& seed_sets, const std::vector& seeds, const VectorView& minimizers) const; - /// Struct to represent per-DP-method stats. + /// Per-thread stage timing accumulators. Must be resized to thread_count before mapping. + std::vector stage_timings_by_thread; + + /// Struct to represent per-DP-method stats. struct aligner_stats_t { /// Collection of values you can += diff --git a/src/minimizer_mapper_from_chains.cpp b/src/minimizer_mapper_from_chains.cpp index 26c86dce02..e805ee2454 100644 --- a/src/minimizer_mapper_from_chains.cpp +++ b/src/minimizer_mapper_from_chains.cpp @@ -29,10 +29,13 @@ #include #include +#include #include #include #include +#include + // Turn on debugging prints //#define debug // Turn on recombintion debugging prints @@ -697,8 +700,20 @@ vector MinimizerMapper::map_from_chains(Alignment& aln) { return aln.sequence(); }); + // Set up per-stage timing. + stage_timings_t* per_thread_timings = nullptr; + if (!stage_timings_by_thread.empty()) { + per_thread_timings = &stage_timings_by_thread[omp_get_thread_num()]; + } + auto stage_clock_now = []() { return std::chrono::high_resolution_clock::now(); }; + auto stage_ns = [](std::chrono::high_resolution_clock::time_point s, + std::chrono::high_resolution_clock::time_point e) { + return std::chrono::duration(e - s).count(); + }; + auto total_start = stage_clock_now(); // Minimizers sorted by position + auto minimizer_stage_start = stage_clock_now(); std::vector minimizers_in_read = this->find_minimizers(aln.sequence(), funnel); // Flag minimizers as being in repetitive regions of the read or not this->flag_repetitive_minimizers(minimizers_in_read); @@ -706,10 +721,12 @@ vector MinimizerMapper::map_from_chains(Alignment& aln) { std::vector minimizer_score_order = sort_minimizers_by_score(minimizers_in_read, rng); // Minimizers sorted by best score first VectorView minimizers{minimizers_in_read, minimizer_score_order}; - + if (per_thread_timings) { per_thread_timings->minimizer_ns += stage_ns(minimizer_stage_start, stage_clock_now()); } // Find the seeds and mark the minimizers that were located. + auto seed_stage_start = stage_clock_now(); vector seeds = this->find_seeds(minimizers_in_read, minimizers, aln, funnel); + if (per_thread_timings) { per_thread_timings->seed_ns += stage_ns(seed_stage_start, stage_clock_now()); } if (seeds.empty()) { #pragma omp critical (cerr) @@ -721,6 +738,7 @@ vector MinimizerMapper::map_from_chains(Alignment& aln) { } // Make them into a zip code tree + auto tree_stage_start = stage_clock_now(); ZipCodeForest zip_code_forest; crash_unless(distance_index); zip_code_forest.fill_in_forest(seeds, *distance_index, aln.sequence().size() * zipcode_tree_scale); @@ -739,6 +757,7 @@ vector MinimizerMapper::map_from_chains(Alignment& aln) { // use them to make gapless extension anchors over them. // TODO: Can we only use the seeds that are in trees we keep? vector seed_anchors = this->to_anchors(aln, minimizers, seeds); + if (per_thread_timings) { per_thread_timings->tree_ns += stage_ns(tree_stage_start, stage_clock_now()); } // If we do gapless extension, then it is possible to find full-length gapless extensions at this stage // If we have at least one good gapless extension, then we will turn them directly into alignments @@ -765,11 +784,13 @@ vector MinimizerMapper::map_from_chains(Alignment& aln) { // The multiplicity for each chain. For now, just the multiplicity of the tree it came from std::vector multiplicity_by_chain; + auto chain_stage_start = stage_clock_now(); do_chaining_on_trees(aln, zip_code_forest, seeds, minimizers, seed_anchors, chains, chain_rec_flags, chain_source_tree, chain_score_estimates, minimizer_kept_chain_count, multiplicity_by_chain, alignments, minimizer_explored, multiplicity_by_alignment, rng, funnel); + if (per_thread_timings) { per_thread_timings->chain_ns += stage_ns(chain_stage_start, stage_clock_now()); } //Fill in chain stats for annotating the final alignment bool best_chain_correct = false; @@ -821,26 +842,29 @@ vector MinimizerMapper::map_from_chains(Alignment& aln) { alignments_to_source.reserve(chain_score_estimates.size()); if (alignments.size() == 0) { - do_alignment_on_chains(aln, seeds, minimizers, seed_anchors, chains, chain_source_tree, multiplicity_by_chain, chain_score_estimates, - minimizer_kept_chain_count, alignments, multiplicity_by_alignment, + auto align_stage_start = stage_clock_now(); + do_alignment_on_chains(aln, seeds, minimizers, seed_anchors, chains, chain_source_tree, multiplicity_by_chain, chain_score_estimates, + minimizer_kept_chain_count, alignments, multiplicity_by_alignment, alignments_to_source, minimizer_explored, stats, funnel_depleted, rng, funnel); + if (per_thread_timings) { per_thread_timings->align_ns += stage_ns(align_stage_start, stage_clock_now()); } } - - + + if (track_provenance) { // Now say we are finding the winner(s) funnel.stage("winner"); } // Fill this in with the alignments we will output as mappings + auto winner_stage_start = stage_clock_now(); vector mappings; mappings.reserve(min(alignments.size(), max_multimaps)); //The scores of the mappings vector scores; //The multiplicities of mappings vector multiplicity_by_mapping; - - pick_mappings_from_alignments(aln, alignments, multiplicity_by_alignment, alignments_to_source, chain_score_estimates, + + pick_mappings_from_alignments(aln, alignments, multiplicity_by_alignment, alignments_to_source, chain_score_estimates, mappings, scores, multiplicity_by_mapping, funnel_depleted, rng, funnel); if (track_provenance) { @@ -1072,6 +1096,13 @@ vector MinimizerMapper::map_from_chains(Alignment& aln) { Explainer::clear_context(); + // Record winner stage and total map_from_chains time. + if (per_thread_timings) { + per_thread_timings->winner_ns += stage_ns(winner_stage_start, stage_clock_now()); + per_thread_timings->total_ns += stage_ns(total_start, stage_clock_now()); + per_thread_timings->read_count++; + } + return mappings; } diff --git a/src/subcommand/giraffe_main.cpp b/src/subcommand/giraffe_main.cpp index f23fead27f..d233c84fc1 100644 --- a/src/subcommand/giraffe_main.cpp +++ b/src/subcommand/giraffe_main.cpp @@ -7,6 +7,7 @@ #include #include #include +#include #include #include #include @@ -1967,6 +1968,9 @@ int main_giraffe(int argc, char** argv) { // Set up counters per-thread for total reads mapped vector reads_mapped_by_thread(thread_count, 0); + + // Initialize per-thread stage timing accumulators. + minimizer_mapper.initialize_stage_timings(thread_count); // For timing, we may run one thread first and then switch to all threads. So track both start times. std::chrono::time_point first_thread_start; @@ -2414,6 +2418,28 @@ int main_giraffe(int argc, char** argv) { } logger.info() << "Memory footprint: " << gbwt::inGigabytes(gbwt::memoryUsage()) << " GB" << endl; + + // Report per-stage timing breakdown. + auto timings = minimizer_mapper.get_stage_timings(); + if (timings.read_count > 0) { + double rc = (double)timings.read_count; + // Convert nanoseconds per read to milliseconds. + auto ms = [&](double ns) { return ns / rc / 1e6; }; + double total_ms = timings.total_ns / rc / 1e6; + auto pct = [&](double ns) -> double { + return total_ms > 0 ? (ns / rc / 1e6) / total_ms * 100.0 : 0.0; + }; + auto li = logger.info(); + li << "Stage timing breakdown (avg per read across " << timings.read_count << " reads):\n"; + li << std::fixed << std::setprecision(3); + li << " minimizer: " << std::setw(8) << ms(timings.minimizer_ns) << " ms (" << std::setw(5) << std::setprecision(1) << pct(timings.minimizer_ns) << "%)\n"; + li << " seed: " << std::setw(8) << std::setprecision(3) << ms(timings.seed_ns) << " ms (" << std::setw(5) << std::setprecision(1) << pct(timings.seed_ns) << "%)\n"; + li << " tree: " << std::setw(8) << std::setprecision(3) << ms(timings.tree_ns) << " ms (" << std::setw(5) << std::setprecision(1) << pct(timings.tree_ns) << "%)\n"; + li << " chain: " << std::setw(8) << std::setprecision(3) << ms(timings.chain_ns) << " ms (" << std::setw(5) << std::setprecision(1) << pct(timings.chain_ns) << "%)\n"; + li << " align: " << std::setw(8) << std::setprecision(3) << ms(timings.align_ns) << " ms (" << std::setw(5) << std::setprecision(1) << pct(timings.align_ns) << "%)\n"; + li << " winner: " << std::setw(8) << std::setprecision(3) << ms(timings.winner_ns) << " ms (" << std::setw(5) << std::setprecision(1) << pct(timings.winner_ns) << "%)\n"; + li << " total: " << std::setw(8) << std::setprecision(3) << total_ms << " ms (100.0%)" << endl; + } } From e5513aa27d9f71635f9a172bd6cf38d714896b09 Mon Sep 17 00:00:00 2001 From: Zia <194475824+electricEpilith@users.noreply.github.com> Date: Sun, 5 Apr 2026 19:49:51 -0700 Subject: [PATCH 66/77] Revert "second try at instrumentation" This reverts commit 6cf266c9bf6ccc362489d88595563fd196a47613. --- src/minimizer_mapper.cpp | 35 ++-------------------- src/minimizer_mapper.hpp | 45 ++-------------------------- src/minimizer_mapper_from_chains.cpp | 45 +++++----------------------- src/subcommand/giraffe_main.cpp | 26 ---------------- 4 files changed, 11 insertions(+), 140 deletions(-) diff --git a/src/minimizer_mapper.cpp b/src/minimizer_mapper.cpp index c1e5744cb0..c678d3ecdc 100644 --- a/src/minimizer_mapper.cpp +++ b/src/minimizer_mapper.cpp @@ -30,12 +30,9 @@ #include #include -#include #include #include -#include - // Turn on debugging prints //#define debug // Turn on printing of minimizer fact tables @@ -624,34 +621,17 @@ vector MinimizerMapper::map_from_extensions(Alignment& aln) { return aln.sequence(); }); - // Set up per-stage timing. - stage_timings_t* per_thread_timings = nullptr; - if (!stage_timings_by_thread.empty()) { - per_thread_timings = &stage_timings_by_thread[omp_get_thread_num()]; - } - auto stage_clock_now = []() { return std::chrono::high_resolution_clock::now(); }; - auto stage_ns = [](std::chrono::high_resolution_clock::time_point s, - std::chrono::high_resolution_clock::time_point e) { - return std::chrono::duration(e - s).count(); - }; - auto total_start = stage_clock_now(); - // Minimizers sorted by position - auto minimizer_stage_start = stage_clock_now(); std::vector minimizers_in_read = this->find_minimizers(aln.sequence(), funnel); // Indexes of minimizers, sorted into score order, best score first std::vector minimizer_score_order = sort_minimizers_by_score(minimizers_in_read, rng); // Minimizers sorted by best score first VectorView minimizers{minimizers_in_read, minimizer_score_order}; - if (per_thread_timings) { per_thread_timings->minimizer_ns += stage_ns(minimizer_stage_start, stage_clock_now()); } // Find the seeds and mark the minimizers that were located. - auto seed_stage_start = stage_clock_now(); vector seeds = this->find_seeds(minimizers_in_read, minimizers, aln, funnel); - if (per_thread_timings) { per_thread_timings->seed_ns += stage_ns(seed_stage_start, stage_clock_now()); } - + // Cluster the seeds. Get sets of input seed indexes that go together. - auto tree_stage_start = stage_clock_now(); // "tree" slot holds cluster+extend for extensions path if (track_provenance) { funnel.stage("cluster"); } @@ -852,8 +832,6 @@ vector MinimizerMapper::map_from_extensions(Alignment& aln) { }); std::vector cluster_extension_scores = this->score_extensions(cluster_extensions, aln, funnel); - if (per_thread_timings) { per_thread_timings->chain_ns += stage_ns(tree_stage_start, stage_clock_now()); } - auto align_stage_start = stage_clock_now(); if (track_provenance) { funnel.stage("align"); } @@ -1095,13 +1073,11 @@ vector MinimizerMapper::map_from_extensions(Alignment& aln) { supplementaries = std::move(identify_supplementary_alignments(alignments, funnel)); } - if (per_thread_timings) { per_thread_timings->align_ns += stage_ns(align_stage_start, stage_clock_now()); } - auto winner_stage_start = stage_clock_now(); if (track_provenance) { // Now say we are finding the winner(s) funnel.stage("winner"); } - + // Fill this in with the alignments we will output as mappings vector mappings; mappings.reserve(min(alignments.size(), max_multimaps)); @@ -1298,13 +1274,6 @@ vector MinimizerMapper::map_from_extensions(Alignment& aln) { } } - // Record winner stage and total map_from_extensions time. - if (per_thread_timings) { - per_thread_timings->winner_ns += stage_ns(winner_stage_start, stage_clock_now()); - per_thread_timings->total_ns += stage_ns(total_start, stage_clock_now()); - per_thread_timings->read_count++; - } - return mappings; } diff --git a/src/minimizer_mapper.hpp b/src/minimizer_mapper.hpp index e3dd804ba6..41a376069e 100644 --- a/src/minimizer_mapper.hpp +++ b/src/minimizer_mapper.hpp @@ -590,45 +590,7 @@ class MinimizerMapper : public AlignerClient { return this->occs.second; } }; - - /// Struct for per-stage timing accumulators - struct stage_timings_t { - double minimizer_ns = 0; ///< find_minimizers + flag_repetitive + sort - double seed_ns = 0; ///< find_seeds - double tree_ns = 0; ///< zip code forest fill + to_anchors + tree scoring - double chain_ns = 0; ///< do_chaining_on_trees (incl. gapless extension) - double align_ns = 0; ///< do_alignment_on_chains - double winner_ns = 0; ///< pick_mappings + MAPQ + annotation - double total_ns = 0; ///< full map_from_chains/map_from_extensions wall time - size_t read_count = 0; - - inline stage_timings_t& operator+=(const stage_timings_t& other) { - minimizer_ns += other.minimizer_ns; - seed_ns += other.seed_ns; - tree_ns += other.tree_ns; - chain_ns += other.chain_ns; - align_ns += other.align_ns; - winner_ns += other.winner_ns; - total_ns += other.total_ns; - read_count += other.read_count; - return *this; - } - }; - - /// Initialize per-thread stage timing accumulators for the given number of threads - void initialize_stage_timings(size_t thread_count) { - stage_timings_by_thread.assign(thread_count, stage_timings_t{}); - } - - /// Aggregate stage timings across all threads. - stage_timings_t get_stage_timings() const { - stage_timings_t total; - for (auto& t : stage_timings_by_thread) { - total += t; - } - return total; - } - + protected: /// Types of paired alignments in paired-end mapping @@ -822,10 +784,7 @@ class MinimizerMapper : public AlignerClient { */ double get_read_coverage(const Alignment& aln, const VectorView>& seed_sets, const std::vector& seeds, const VectorView& minimizers) const; - /// Per-thread stage timing accumulators. Must be resized to thread_count before mapping. - std::vector stage_timings_by_thread; - - /// Struct to represent per-DP-method stats. + /// Struct to represent per-DP-method stats. struct aligner_stats_t { /// Collection of values you can += diff --git a/src/minimizer_mapper_from_chains.cpp b/src/minimizer_mapper_from_chains.cpp index e805ee2454..26c86dce02 100644 --- a/src/minimizer_mapper_from_chains.cpp +++ b/src/minimizer_mapper_from_chains.cpp @@ -29,13 +29,10 @@ #include #include -#include #include #include #include -#include - // Turn on debugging prints //#define debug // Turn on recombintion debugging prints @@ -700,20 +697,8 @@ vector MinimizerMapper::map_from_chains(Alignment& aln) { return aln.sequence(); }); - // Set up per-stage timing. - stage_timings_t* per_thread_timings = nullptr; - if (!stage_timings_by_thread.empty()) { - per_thread_timings = &stage_timings_by_thread[omp_get_thread_num()]; - } - auto stage_clock_now = []() { return std::chrono::high_resolution_clock::now(); }; - auto stage_ns = [](std::chrono::high_resolution_clock::time_point s, - std::chrono::high_resolution_clock::time_point e) { - return std::chrono::duration(e - s).count(); - }; - auto total_start = stage_clock_now(); // Minimizers sorted by position - auto minimizer_stage_start = stage_clock_now(); std::vector minimizers_in_read = this->find_minimizers(aln.sequence(), funnel); // Flag minimizers as being in repetitive regions of the read or not this->flag_repetitive_minimizers(minimizers_in_read); @@ -721,12 +706,10 @@ vector MinimizerMapper::map_from_chains(Alignment& aln) { std::vector minimizer_score_order = sort_minimizers_by_score(minimizers_in_read, rng); // Minimizers sorted by best score first VectorView minimizers{minimizers_in_read, minimizer_score_order}; - if (per_thread_timings) { per_thread_timings->minimizer_ns += stage_ns(minimizer_stage_start, stage_clock_now()); } + // Find the seeds and mark the minimizers that were located. - auto seed_stage_start = stage_clock_now(); vector seeds = this->find_seeds(minimizers_in_read, minimizers, aln, funnel); - if (per_thread_timings) { per_thread_timings->seed_ns += stage_ns(seed_stage_start, stage_clock_now()); } if (seeds.empty()) { #pragma omp critical (cerr) @@ -738,7 +721,6 @@ vector MinimizerMapper::map_from_chains(Alignment& aln) { } // Make them into a zip code tree - auto tree_stage_start = stage_clock_now(); ZipCodeForest zip_code_forest; crash_unless(distance_index); zip_code_forest.fill_in_forest(seeds, *distance_index, aln.sequence().size() * zipcode_tree_scale); @@ -757,7 +739,6 @@ vector MinimizerMapper::map_from_chains(Alignment& aln) { // use them to make gapless extension anchors over them. // TODO: Can we only use the seeds that are in trees we keep? vector seed_anchors = this->to_anchors(aln, minimizers, seeds); - if (per_thread_timings) { per_thread_timings->tree_ns += stage_ns(tree_stage_start, stage_clock_now()); } // If we do gapless extension, then it is possible to find full-length gapless extensions at this stage // If we have at least one good gapless extension, then we will turn them directly into alignments @@ -784,13 +765,11 @@ vector MinimizerMapper::map_from_chains(Alignment& aln) { // The multiplicity for each chain. For now, just the multiplicity of the tree it came from std::vector multiplicity_by_chain; - auto chain_stage_start = stage_clock_now(); do_chaining_on_trees(aln, zip_code_forest, seeds, minimizers, seed_anchors, chains, chain_rec_flags, chain_source_tree, chain_score_estimates, minimizer_kept_chain_count, multiplicity_by_chain, alignments, minimizer_explored, multiplicity_by_alignment, rng, funnel); - if (per_thread_timings) { per_thread_timings->chain_ns += stage_ns(chain_stage_start, stage_clock_now()); } //Fill in chain stats for annotating the final alignment bool best_chain_correct = false; @@ -842,29 +821,26 @@ vector MinimizerMapper::map_from_chains(Alignment& aln) { alignments_to_source.reserve(chain_score_estimates.size()); if (alignments.size() == 0) { - auto align_stage_start = stage_clock_now(); - do_alignment_on_chains(aln, seeds, minimizers, seed_anchors, chains, chain_source_tree, multiplicity_by_chain, chain_score_estimates, - minimizer_kept_chain_count, alignments, multiplicity_by_alignment, + do_alignment_on_chains(aln, seeds, minimizers, seed_anchors, chains, chain_source_tree, multiplicity_by_chain, chain_score_estimates, + minimizer_kept_chain_count, alignments, multiplicity_by_alignment, alignments_to_source, minimizer_explored, stats, funnel_depleted, rng, funnel); - if (per_thread_timings) { per_thread_timings->align_ns += stage_ns(align_stage_start, stage_clock_now()); } } - - + + if (track_provenance) { // Now say we are finding the winner(s) funnel.stage("winner"); } // Fill this in with the alignments we will output as mappings - auto winner_stage_start = stage_clock_now(); vector mappings; mappings.reserve(min(alignments.size(), max_multimaps)); //The scores of the mappings vector scores; //The multiplicities of mappings vector multiplicity_by_mapping; - - pick_mappings_from_alignments(aln, alignments, multiplicity_by_alignment, alignments_to_source, chain_score_estimates, + + pick_mappings_from_alignments(aln, alignments, multiplicity_by_alignment, alignments_to_source, chain_score_estimates, mappings, scores, multiplicity_by_mapping, funnel_depleted, rng, funnel); if (track_provenance) { @@ -1096,13 +1072,6 @@ vector MinimizerMapper::map_from_chains(Alignment& aln) { Explainer::clear_context(); - // Record winner stage and total map_from_chains time. - if (per_thread_timings) { - per_thread_timings->winner_ns += stage_ns(winner_stage_start, stage_clock_now()); - per_thread_timings->total_ns += stage_ns(total_start, stage_clock_now()); - per_thread_timings->read_count++; - } - return mappings; } diff --git a/src/subcommand/giraffe_main.cpp b/src/subcommand/giraffe_main.cpp index d233c84fc1..f23fead27f 100644 --- a/src/subcommand/giraffe_main.cpp +++ b/src/subcommand/giraffe_main.cpp @@ -7,7 +7,6 @@ #include #include #include -#include #include #include #include @@ -1968,9 +1967,6 @@ int main_giraffe(int argc, char** argv) { // Set up counters per-thread for total reads mapped vector reads_mapped_by_thread(thread_count, 0); - - // Initialize per-thread stage timing accumulators. - minimizer_mapper.initialize_stage_timings(thread_count); // For timing, we may run one thread first and then switch to all threads. So track both start times. std::chrono::time_point first_thread_start; @@ -2418,28 +2414,6 @@ int main_giraffe(int argc, char** argv) { } logger.info() << "Memory footprint: " << gbwt::inGigabytes(gbwt::memoryUsage()) << " GB" << endl; - - // Report per-stage timing breakdown. - auto timings = minimizer_mapper.get_stage_timings(); - if (timings.read_count > 0) { - double rc = (double)timings.read_count; - // Convert nanoseconds per read to milliseconds. - auto ms = [&](double ns) { return ns / rc / 1e6; }; - double total_ms = timings.total_ns / rc / 1e6; - auto pct = [&](double ns) -> double { - return total_ms > 0 ? (ns / rc / 1e6) / total_ms * 100.0 : 0.0; - }; - auto li = logger.info(); - li << "Stage timing breakdown (avg per read across " << timings.read_count << " reads):\n"; - li << std::fixed << std::setprecision(3); - li << " minimizer: " << std::setw(8) << ms(timings.minimizer_ns) << " ms (" << std::setw(5) << std::setprecision(1) << pct(timings.minimizer_ns) << "%)\n"; - li << " seed: " << std::setw(8) << std::setprecision(3) << ms(timings.seed_ns) << " ms (" << std::setw(5) << std::setprecision(1) << pct(timings.seed_ns) << "%)\n"; - li << " tree: " << std::setw(8) << std::setprecision(3) << ms(timings.tree_ns) << " ms (" << std::setw(5) << std::setprecision(1) << pct(timings.tree_ns) << "%)\n"; - li << " chain: " << std::setw(8) << std::setprecision(3) << ms(timings.chain_ns) << " ms (" << std::setw(5) << std::setprecision(1) << pct(timings.chain_ns) << "%)\n"; - li << " align: " << std::setw(8) << std::setprecision(3) << ms(timings.align_ns) << " ms (" << std::setw(5) << std::setprecision(1) << pct(timings.align_ns) << "%)\n"; - li << " winner: " << std::setw(8) << std::setprecision(3) << ms(timings.winner_ns) << " ms (" << std::setw(5) << std::setprecision(1) << pct(timings.winner_ns) << "%)\n"; - li << " total: " << std::setw(8) << std::setprecision(3) << total_ms << " ms (100.0%)" << endl; - } } From e9c3d4002add16c1dc8e54ed8e878045ce32dda8 Mon Sep 17 00:00:00 2001 From: Zia Truong <194475824+electricEpilith@users.noreply.github.com> Date: Tue, 7 Apr 2026 12:32:50 -0700 Subject: [PATCH 67/77] snarl distance index version number update --- deps/libbdsg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deps/libbdsg b/deps/libbdsg index ea70e5575b..09f9d6ed93 160000 --- a/deps/libbdsg +++ b/deps/libbdsg @@ -1 +1 @@ -Subproject commit ea70e5575b37afe3866b217ba87e362542635307 +Subproject commit 09f9d6ed933aa7cf4ff54efbc4f04268443f9dba From 848d4b929cb82ecef556c589221811ec33d01693 Mon Sep 17 00:00:00 2001 From: Zia <194475824+electricEpilith@users.noreply.github.com> Date: Tue, 7 Apr 2026 16:06:10 -0700 Subject: [PATCH 68/77] adapter draft (needs review) Co-Authored-By: Claude Sonnet 4.6 --- .gitmodules | 4 + deps/theseus-lib | 1 + src/handle_graph_theseus_adapter.hpp | 110 +++++++++++++++++++++++++++ 3 files changed, 115 insertions(+) create mode 160000 deps/theseus-lib create mode 100644 src/handle_graph_theseus_adapter.hpp diff --git a/.gitmodules b/.gitmodules index 1bb80d2346..f485142d0e 100644 --- a/.gitmodules +++ b/.gitmodules @@ -133,3 +133,7 @@ [submodule "deps/mimalloc"] path = deps/mimalloc url = https://github.com/microsoft/mimalloc.git +[submodule "deps/theseus-lib"] + path = deps/theseus-lib + url = https://github.com/albertjimenezbl/theseus-lib.git + branch = vg-integration diff --git a/deps/theseus-lib b/deps/theseus-lib new file mode 160000 index 0000000000..05277907e5 --- /dev/null +++ b/deps/theseus-lib @@ -0,0 +1 @@ +Subproject commit 05277907e5a5901002699eb43dfb4efef4f3acf9 diff --git a/src/handle_graph_theseus_adapter.hpp b/src/handle_graph_theseus_adapter.hpp new file mode 100644 index 0000000000..2dd20f3329 --- /dev/null +++ b/src/handle_graph_theseus_adapter.hpp @@ -0,0 +1,110 @@ +#pragma once + +#include +#include +#include + +#include +#include "theseus/graph.h" + +/** + * @file handle_graph_theseus_adapter.hpp + * + * Builds a theseus::Graph from a HandleGraph and maintains a bidirectional + * mapping between theseus integer vertex IDs and handle_t values. + * + * Why this exists instead of theseus::Graph(HandleGraph): + * - The upstream constructor has a naming collision bug: both orientations of + * a node get the same name because get_id() strips the orientation bit, + * causing silent overwrites in name_to_id_. + * - After alignment, theseus reports results as integer vertex IDs with no + * built-in path back to handle_t. + * + * Vertex naming convention: "+" for forward, "-" for reverse. + * Use handle_name() to produce the start_node string required by align(). + * + * Usage: + * HandleGraphTheseusAdapter adapter(my_handle_graph); + * TheseusAlignerImpl impl(penalties, adapter.take_graph(), false); + * auto aln = impl.align(seq, adapter.handle_name(start_handle)); + * handle_t result_handle = adapter.vertex_to_handle(aln.some_vertex_id); + */ + +namespace vg { + +class HandleGraphTheseusAdapter { +public: + explicit HandleGraphTheseusAdapter(const handlegraph::HandleGraph& hg) { + const size_t n = hg.get_node_count(); + graph_._vertices.reserve(2 * n); + graph_.name_to_id_.reserve(2 * n); + id_to_handle_.reserve(2 * n); + handle_to_id_.reserve(2 * n); + + // Add a vertex for each handle orientation and record the mapping. + hg.for_each_handle([&](const handlegraph::handle_t& h) { + add_vertex(hg, h); + add_vertex(hg, hg.flip(h)); + }); + + // for_each_edge visits each edge once in canonical form (first.id <= + // second.id). Add the forward traversal and its reverse complement. + hg.for_each_edge([&](const handlegraph::edge_t& e) { + add_edge(e.first, e.second); + // Reverse complement: flip both endpoints and swap direction. + add_edge(hg.flip(e.second), hg.flip(e.first)); + }); + } + + // Move the built graph out for use with TheseusAlignerImpl. + // The adapter retains the handle<->vertex mappings; only call this once. + theseus::Graph&& take_graph() { return std::move(graph_); } + + // The vertex name string to pass as start_node to TheseusAlignerImpl::align(). + static std::string handle_name(const handlegraph::HandleGraph& hg, + const handlegraph::handle_t& h) { + return std::to_string(hg.get_id(h)) + + (hg.get_is_reverse(h) ? '-' : '+'); + } + + // Convert a theseus vertex index (from alignment results) back to a handle_t. + handlegraph::handle_t vertex_to_handle(int vertex_id) const { + return id_to_handle_.at(static_cast(vertex_id)); + } + + // Convert a handle_t to its theseus vertex index. + int handle_to_vertex(const handlegraph::handle_t& h) const { + return handle_to_id_.at(h); + } + +private: + theseus::Graph graph_; + std::vector id_to_handle_; + std::unordered_map handle_to_id_; + + void add_vertex(const handlegraph::HandleGraph& hg, + const handlegraph::handle_t& h) { + const int idx = static_cast(graph_._vertices.size()); + theseus::Graph::vertex v; + v.name = handle_name(hg, h); + v.value = hg.get_sequence(h); // sequence in handle's orientation + graph_._vertices.push_back(std::move(v)); + graph_.name_to_id_[graph_._vertices.back().name] = idx; + id_to_handle_.push_back(h); + handle_to_id_[h] = idx; + } + + void add_edge(const handlegraph::handle_t& from, + const handlegraph::handle_t& to) { + const int from_idx = handle_to_id_.at(from); + const int to_idx = handle_to_id_.at(to); + theseus::Graph::edge e; + e.from_vertex = from_idx; + e.to_vertex = to_idx; + e.overlap = 0; + graph_._vertices[from_idx].out_edges.push_back(e); + graph_._vertices[to_idx].in_edges.push_back(e); + } +}; + +} // namespace vg From d821b836d5ae7fad99835e5f904de8959ec6e58c Mon Sep 17 00:00:00 2001 From: Zia <194475824+electricEpilith@users.noreply.github.com> Date: Tue, 7 Apr 2026 16:40:16 -0700 Subject: [PATCH 69/77] include theseus-lib Co-Authored-By: Claude Sonnet 4.6 --- Makefile | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 6159297339..27d796d337 100644 --- a/Makefile +++ b/Makefile @@ -74,7 +74,7 @@ INCLUDE_FLAGS :=-I$(CWD)/$(INC_DIR) -I. -I$(CWD)/$(SRC_DIR) -I$(CWD)/$(UNITTEST_ # These need to come before library search paths from LDFLAGS or we won't # prefer linking vg-installed dependencies over system ones. LD_LIB_DIR_FLAGS := -L$(CWD)/$(LIB_DIR) -LD_LIB_FLAGS := -lvcflib -lwfa2 -ltabixpp -lgssw -lssw -lsublinearLS -lpthread -lncurses -lgcsa2 -lgbwtgraph -lgbwt -lkff -ldivsufsort -ldivsufsort64 -lvcfh -lraptor2 -lpinchesandcacti -l3edgeconnected -lsonlib -lfml -lstructures -lbdsg -lxg -lsdsl -lzstd -lhandlegraph -lcrypto +LD_LIB_FLAGS := -lvcflib -lwfa2 -ltabixpp -lgssw -lssw -lsublinearLS -lpthread -lncurses -lgcsa2 -lgbwtgraph -lgbwt -lkff -ldivsufsort -ldivsufsort64 -lvcfh -lraptor2 -lpinchesandcacti -l3edgeconnected -lsonlib -lfml -lstructures -lbdsg -lxg -lsdsl -lzstd -ltheseus -lhandlegraph -lcrypto # We omit Boost Program Options for now; we find it in a platform-dependent way. # By default it has no suffix BOOST_SUFFIX="" @@ -374,6 +374,7 @@ IPS4O_DIR=deps/ips4o BBHASH_DIR=deps/BBHash MIO_DIR=deps/mio ATOMIC_QUEUE_DIR=deps/atomic_queue +THESEUS_DIR:=deps/theseus-lib # Dependencies that go into libvg's archive # These go in libvg but come from dependencies @@ -410,6 +411,7 @@ LIB_DEPS += $(LIB_DIR)/libvgio.a LIB_DEPS += $(LIB_DIR)/libhandlegraph.a LIB_DEPS += $(LIB_DIR)/libbdsg.a LIB_DEPS += $(LIB_DIR)/libxg.a +LIB_DEPS += $(LIB_DIR)/libtheseus.a ifneq ($(shell uname -s),Darwin) # On non-Mac (i.e. Linux), where ELF binaries are used, pull in libdw which # backward-cpp will use. @@ -936,6 +938,10 @@ $(LIB_DIR)/libxg.a: $(XG_DIR)/src/*.hpp $(XG_DIR)/src/*.cpp $(INC_DIR)/mmmultima +$(CXX) $(INCLUDE_FLAGS) $(CXXFLAGS) $(CPPFLAGS) -fPIC -DNO_GFAKLUGE -c -o $(XG_DIR)/xg.o $(XG_DIR)/src/xg.cpp $(FILTER) +ar rs $@ $(XG_DIR)/xg.o +$(LIB_DIR)/libtheseus.a: $(LIB_DIR)/libhandlegraph.a $(wildcard $(THESEUS_DIR)/theseus/*.cpp) $(wildcard $(THESEUS_DIR)/include/theseus/*.h) + +rm -Rf $(CWD)/$(INC_DIR)/theseus + +cd $(THESEUS_DIR) && rm -Rf build && mkdir build && cd build && cmake -DCMAKE_C_COMPILER="$(CC)" -DCMAKE_CXX_COMPILER="$(CXX)" -DCMAKE_VERBOSE_MAKEFILE=ON -DCMAKE_C_FLAGS="-fPIC $(CFLAGS)" -DCMAKE_CXX_FLAGS="-fPIC $(CPPFLAGS)" -DCMAKE_BUILD_TYPE=Release -DENABLE_TESTS=OFF -DFETCHCONTENT_SOURCE_DIR_LIBHANDLEGRAPH=$(CWD)/$(LIBHANDLEGRAPH_DIR) .. $(FILTER) && $(MAKE) theseus $(FILTER) && cp libtheseus.a $(CWD)/$(LIB_DIR)/ && cp -r $(CWD)/$(THESEUS_DIR)/include/theseus $(CWD)/$(INC_DIR)/ + # Auto-git-versioning # Can be overridden from the environment to supply a version if none is on disk. @@ -1157,6 +1163,7 @@ clean: clean-vcflib cd $(DEP_DIR) && cd sublinear-Li-Stephens && $(MAKE) clean cd $(DEP_DIR) && cd libhandlegraph && rm -Rf build CMakeCache.txt CMakeFiles cd $(DEP_DIR) && cd libvgio && rm -Rf build CMakeCache.txt CMakeFiles + cd $(DEP_DIR) && cd theseus-lib && rm -Rf build cd $(DEP_DIR) && cd raptor && cd build && find . -not \( -name '.gitignore' -or -name 'pkg.m4' \) -delete # lru_cache is never built because it is header-only # bash-tap is never built either From dcb296590e48419d0a385fb560f1f9bb909bd74e Mon Sep 17 00:00:00 2001 From: Zia <194475824+electricEpilith@users.noreply.github.com> Date: Tue, 7 Apr 2026 17:40:06 -0700 Subject: [PATCH 70/77] theseus::Alignment -> vg::Alignment converter draft Co-Authored-By: Claude Sonnet 4.6 --- src/handle_graph_theseus_adapter.hpp | 122 +++++++++++++++++++++++++++ 1 file changed, 122 insertions(+) diff --git a/src/handle_graph_theseus_adapter.hpp b/src/handle_graph_theseus_adapter.hpp index 2dd20f3329..53abe6c3b8 100644 --- a/src/handle_graph_theseus_adapter.hpp +++ b/src/handle_graph_theseus_adapter.hpp @@ -6,6 +6,8 @@ #include #include "theseus/graph.h" +#include "theseus/alignment.h" +#include /** * @file handle_graph_theseus_adapter.hpp @@ -107,4 +109,124 @@ class HandleGraphTheseusAdapter { } }; +/** + * @brief Convert a theseus::Alignment to a vg::Alignment. + * + * theseus edit operations are per-base characters: + * 'M' = match, 'X' = mismatch/SNP, 'I' = insertion, 'D' = deletion. + * Consecutive operations of the same type are merged into a single vg Edit. + * Edits are split at node boundaries according to the reference bases consumed + * in each vertex of the theseus path. + * + * @param theseus_aln Alignment returned by TheseusAlignerImpl::align(). + * @param sequence The read sequence that was aligned. + * @param adapter Adapter used to map theseus vertex IDs → handle_t. + * @param graph The HandleGraph the alignment was made against. + * @return vg::Alignment with sequence, path, and per-node mappings filled in. + */ +inline vg::Alignment theseus_to_vg_alignment( + const theseus::Alignment& theseus_aln, + const std::string& sequence, + const HandleGraphTheseusAdapter& adapter, + const handlegraph::HandleGraph& graph) { + + vg::Alignment vg_aln; + vg_aln.set_sequence(sequence); + + if (theseus_aln.path.empty() || theseus_aln.edit_op.empty()) { + return vg_aln; + } + + vg::Path* vg_path = vg_aln.mutable_path(); + + const size_t n_edits = theseus_aln.edit_op.size(); + const size_t n_nodes = theseus_aln.path.size(); + size_t edit_cursor = 0; // index into edit_op + size_t seq_cursor = 0; // index into sequence (read bases consumed) + + for (size_t node_idx = 0; node_idx < n_nodes && edit_cursor < n_edits; ++node_idx) { + const handlegraph::handle_t h = adapter.vertex_to_handle(theseus_aln.path[node_idx]); + const size_t node_len = graph.get_length(h); + + // Reference range covered by this alignment on this node. + // start_offset applies only to the first node; + // end_offset (exclusive) applies only to the last node. + const size_t ref_start = (node_idx == 0) + ? static_cast(theseus_aln.start_offset) + : 0; + const size_t ref_end = (node_idx == n_nodes - 1) + ? static_cast(theseus_aln.end_offset) + : node_len; + size_t ref_remaining = ref_end - ref_start; + + vg::Mapping* mapping = vg_path->add_mapping(); + mapping->mutable_position()->set_node_id(graph.get_id(h)); + mapping->mutable_position()->set_is_reverse(graph.get_is_reverse(h)); + mapping->mutable_position()->set_offset(static_cast(ref_start)); + mapping->set_rank(static_cast(node_idx + 1)); + + while (edit_cursor < n_edits) { + const char op = theseus_aln.edit_op[edit_cursor]; + + // Insertions do not consume reference; attach to the current mapping. + if (op == 'I') { + const size_t seq_start = seq_cursor; + size_t run = 0; + while (edit_cursor < n_edits && theseus_aln.edit_op[edit_cursor] == 'I') { + ++run; ++edit_cursor; ++seq_cursor; + } + vg::Edit* e = mapping->add_edit(); + e->set_from_length(0); + e->set_to_length(static_cast(run)); + e->set_sequence(sequence.substr(seq_start, run)); + continue; + } + + // Reference-consuming op: stop if this node is exhausted. + if (ref_remaining == 0) break; + + if (op == 'M') { + size_t run = 0; + while (edit_cursor < n_edits + && theseus_aln.edit_op[edit_cursor] == 'M' + && run < ref_remaining) { + ++run; ++edit_cursor; ++seq_cursor; + } + vg::Edit* e = mapping->add_edit(); + e->set_from_length(static_cast(run)); + e->set_to_length(static_cast(run)); + // no sequence field for matches + ref_remaining -= run; + } else if (op == 'X') { + const size_t seq_start = seq_cursor; + size_t run = 0; + while (edit_cursor < n_edits + && theseus_aln.edit_op[edit_cursor] == 'X' + && run < ref_remaining) { + ++run; ++edit_cursor; ++seq_cursor; + } + vg::Edit* e = mapping->add_edit(); + e->set_from_length(static_cast(run)); + e->set_to_length(static_cast(run)); + e->set_sequence(sequence.substr(seq_start, run)); + ref_remaining -= run; + } else if (op == 'D') { + size_t run = 0; + while (edit_cursor < n_edits + && theseus_aln.edit_op[edit_cursor] == 'D' + && run < ref_remaining) { + ++run; ++edit_cursor; + // deletions consume no read bases + } + vg::Edit* e = mapping->add_edit(); + e->set_from_length(static_cast(run)); + e->set_to_length(0); + ref_remaining -= run; + } + } + } + + return vg_aln; +} + } // namespace vg From 0fe5a602b43fd4d30c9131c4af5ddfa601f05415 Mon Sep 17 00:00:00 2001 From: Zia <194475824+electricEpilith@users.noreply.github.com> Date: Tue, 7 Apr 2026 18:24:12 -0700 Subject: [PATCH 71/77] refactor with_dagified_local_graph, local graph extraction logic moved to get_local_graph --- src/minimizer_mapper.hpp | 5 ++++ src/minimizer_mapper_from_chains.cpp | 42 ++++++++++++++++++---------- 2 files changed, 33 insertions(+), 14 deletions(-) diff --git a/src/minimizer_mapper.hpp b/src/minimizer_mapper.hpp index 41a376069e..a588fb88af 100644 --- a/src/minimizer_mapper.hpp +++ b/src/minimizer_mapper.hpp @@ -1035,6 +1035,11 @@ class MinimizerMapper : public AlignerClient { */ void wfa_alignment_to_alignment(const WFAAlignment& wfa_alignment, Alignment& alignment) const; + /** + * Get the subgraph to align to. + */ + static bdsg::HashGraph get_local_graph(const pos_t& left_anchor, const pos_t& right_anchor, size_t max_path_length, const HandleGraph& graph, unordered_map& local_to_base); + /** * Clip out the part of the graph between the given positions (left facing * into the region to be extracted and right facing out), and dagify it diff --git a/src/minimizer_mapper_from_chains.cpp b/src/minimizer_mapper_from_chains.cpp index 26c86dce02..8ecd067fe1 100644 --- a/src/minimizer_mapper_from_chains.cpp +++ b/src/minimizer_mapper_from_chains.cpp @@ -3125,25 +3125,25 @@ void MinimizerMapper::wfa_alignment_to_alignment(const WFAAlignment& wfa_alignme } } -void MinimizerMapper::with_dagified_local_graph(const pos_t& left_anchor, const pos_t& right_anchor, size_t max_path_length, const HandleGraph& graph, const std::function(const handle_t&)>&)>& callback) { - - if (is_empty(left_anchor) && is_empty(right_anchor)) { - throw ChainAlignmentFailedError("Cannot align sequence between two unset positions"); - } - - // We need to get the graph to align to. +bdsg::HashGraph MinimizerMapper::get_local_graph( + const pos_t& left_anchor, + const pos_t& right_anchor, + size_t max_path_length, + const HandleGraph& graph, + unordered_map& local_to_base +) { bdsg::HashGraph local_graph; - unordered_map local_to_base; + if (!is_empty(left_anchor) && !is_empty(right_anchor)) { // We want a graph actually between two positions. // Enforce strict max length to avoid extra tips. - local_to_base = algorithms::extract_connecting_graph( + local_to_base = std::move(algorithms::extract_connecting_graph( &graph, &local_graph, max_path_length, left_anchor, right_anchor, true - ); + )); if (local_to_base.empty()) { // A possible result is that the one anchor is not reachable from @@ -3161,31 +3161,45 @@ void MinimizerMapper::with_dagified_local_graph(const pos_t& left_anchor, const } } else if (!is_empty(left_anchor)) { // We only have the left anchor - local_to_base = algorithms::extract_extending_graph( + local_to_base = std::move(algorithms::extract_extending_graph( &graph, &local_graph, max_path_length, left_anchor, false, false - ); + )); } else { // We only have the right anchor - local_to_base = algorithms::extract_extending_graph( + local_to_base = std::move(algorithms::extract_extending_graph( &graph, &local_graph, max_path_length, right_anchor, true, false - ); + )); } #ifdef debug std::cerr << "Local graph:" << std::endl; dump_debug_graph(local_graph); #endif + + return local_graph; +} + +void MinimizerMapper::with_dagified_local_graph(const pos_t& left_anchor, const pos_t& right_anchor, size_t max_path_length, const HandleGraph& graph, const std::function(const handle_t&)>&)>& callback) { + + if (is_empty(left_anchor) && is_empty(right_anchor)) { + throw ChainAlignmentFailedError("Cannot align sequence between two unset positions"); + } + // We need to get the graph to align to. + unordered_map local_to_base; + bdsg::HashGraph local_graph = get_local_graph(left_anchor, right_anchor, max_path_length, graph, local_to_base); + + // To find the anchoring nodes in the extracted graph, we need to scan local_to_base. nid_t local_left_anchor_id = 0; nid_t local_right_anchor_id = 0; From 4458a5691872f34abea9686a7a30725a0deee1ad Mon Sep 17 00:00:00 2001 From: Zia <194475824+electricEpilith@users.noreply.github.com> Date: Tue, 7 Apr 2026 18:54:03 -0700 Subject: [PATCH 72/77] fix theseus include Co-Authored-By: Claude Sonnet 4.6 --- Makefile | 1 + 1 file changed, 1 insertion(+) diff --git a/Makefile b/Makefile index 27d796d337..43531d6ff5 100644 --- a/Makefile +++ b/Makefile @@ -375,6 +375,7 @@ BBHASH_DIR=deps/BBHash MIO_DIR=deps/mio ATOMIC_QUEUE_DIR=deps/atomic_queue THESEUS_DIR:=deps/theseus-lib +INCLUDE_FLAGS += -I$(CWD)/$(THESEUS_DIR) # Dependencies that go into libvg's archive # These go in libvg but come from dependencies From 9072c142b70d93202b34a4b06b60743c69260a1a Mon Sep 17 00:00:00 2001 From: Zia <194475824+electricEpilith@users.noreply.github.com> Date: Tue, 7 Apr 2026 19:15:28 -0700 Subject: [PATCH 73/77] start find_chain_alignment_theseus; better file naming --- src/minimizer_mapper.hpp | 5 +++ src/minimizer_mapper_from_chains.cpp | 32 +++++++++++++++++++ ...heseus_adapter.hpp => theseus_interop.hpp} | 6 ++-- 3 files changed, 40 insertions(+), 3 deletions(-) rename src/{handle_graph_theseus_adapter.hpp => theseus_interop.hpp} (98%) diff --git a/src/minimizer_mapper.hpp b/src/minimizer_mapper.hpp index a588fb88af..cef5c04ba7 100644 --- a/src/minimizer_mapper.hpp +++ b/src/minimizer_mapper.hpp @@ -917,6 +917,11 @@ class MinimizerMapper : public AlignerClient { * If given base processing stats for bases and for time, adds aligned bases and consumed time to them. */ Alignment find_chain_alignment(const Alignment& aln, const VectorView& to_chain, const std::vector& chain, aligner_stats_t* stats = nullptr) const; + + /** + * Same as above, but with the Theseus alignment algorithm + */ + Alignment find_chain_alignment_theseus(const Alignment& aln, const VectorView& to_chain, const std::vector& chain, aligner_stats_t* stats = nullptr) const; /** * Operating on the given input alignment, align the tails dangling off the diff --git a/src/minimizer_mapper_from_chains.cpp b/src/minimizer_mapper_from_chains.cpp index 8ecd067fe1..ac2cd0c138 100644 --- a/src/minimizer_mapper_from_chains.cpp +++ b/src/minimizer_mapper_from_chains.cpp @@ -11,6 +11,7 @@ #include "crash.hpp" #include "path_subgraph.hpp" #include "multipath_alignment.hpp" +#include "theseus_interop.hpp" #include "split_strand_graph.hpp" #include "subgraph.hpp" #include "statistics.hpp" @@ -3117,6 +3118,37 @@ Alignment MinimizerMapper::find_chain_alignment( return result; } +Alignment MinimizerMapper::find_chain_alignment_theseus( + const Alignment& aln, + const VectorView& to_chain, + const std::vector& chain, + aligner_stats_t* stats +) const { + + if (chain.empty()) { + throw ChainAlignmentFailedError("Cannot find an alignment for an empty chain!"); + } + + //bdsg::HashGraph local_graph = get_local_graph(); + + + + + // Convert to a vg Alignment. + Alignment result; + /* + // Simplify the path but keep internal deletions; we want to assert the + // read deleted relative to some graph, and avoid jumps along nonexistent + // edges. + *result.mutable_path() = std::move(simplify(composed_path, false)); + result.set_score(composed_score); + if (!result.sequence().empty()) { + result.set_identity(identity(result.path())); + } + */ + return result; +} + void MinimizerMapper::wfa_alignment_to_alignment(const WFAAlignment& wfa_alignment, Alignment& alignment) const { *(alignment.mutable_path()) = wfa_alignment.to_path(this->gbwt_graph, alignment.sequence()); alignment.set_score(wfa_alignment.score); diff --git a/src/handle_graph_theseus_adapter.hpp b/src/theseus_interop.hpp similarity index 98% rename from src/handle_graph_theseus_adapter.hpp rename to src/theseus_interop.hpp index 53abe6c3b8..90806ecb76 100644 --- a/src/handle_graph_theseus_adapter.hpp +++ b/src/theseus_interop.hpp @@ -5,12 +5,12 @@ #include #include -#include "theseus/graph.h" -#include "theseus/alignment.h" +#include +#include #include /** - * @file handle_graph_theseus_adapter.hpp + * @file theseus_interop.hpp * * Builds a theseus::Graph from a HandleGraph and maintains a bidirectional * mapping between theseus integer vertex IDs and handle_t values. From 70887e949045a58966f23b82a722793afd87b4f3 Mon Sep 17 00:00:00 2001 From: Zia <194475824+electricEpilith@users.noreply.github.com> Date: Tue, 7 Apr 2026 19:22:43 -0700 Subject: [PATCH 74/77] update comments, better naming --- src/theseus_interop.hpp | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/src/theseus_interop.hpp b/src/theseus_interop.hpp index 90806ecb76..e8fa5e14b7 100644 --- a/src/theseus_interop.hpp +++ b/src/theseus_interop.hpp @@ -12,8 +12,11 @@ /** * @file theseus_interop.hpp * - * Builds a theseus::Graph from a HandleGraph and maintains a bidirectional - * mapping between theseus integer vertex IDs and handle_t values. + * @brief Interop for using Theseus in vg +*/ + +namespace vg { +/** @brief Builds a theseus::Graph from a HandleGraph and maintains a bidirectional mapping between theseus integer vertex IDs and handle_t values. * * Why this exists instead of theseus::Graph(HandleGraph): * - The upstream constructor has a naming collision bug: both orientations of @@ -26,14 +29,11 @@ * Use handle_name() to produce the start_node string required by align(). * * Usage: - * HandleGraphTheseusAdapter adapter(my_handle_graph); + * HandleGraphTheseusAdapter adapter(handle_graph); * TheseusAlignerImpl impl(penalties, adapter.take_graph(), false); * auto aln = impl.align(seq, adapter.handle_name(start_handle)); * handle_t result_handle = adapter.vertex_to_handle(aln.some_vertex_id); */ - -namespace vg { - class HandleGraphTheseusAdapter { public: explicit HandleGraphTheseusAdapter(const handlegraph::HandleGraph& hg) { @@ -124,7 +124,7 @@ class HandleGraphTheseusAdapter { * @param graph The HandleGraph the alignment was made against. * @return vg::Alignment with sequence, path, and per-node mappings filled in. */ -inline vg::Alignment theseus_to_vg_alignment( +inline vg::Alignment vg_alignment_from_theseus_alignment( const theseus::Alignment& theseus_aln, const std::string& sequence, const HandleGraphTheseusAdapter& adapter, From c7a075a28c9ee03d2e0aeb8aca4edffd8c6e457d Mon Sep 17 00:00:00 2001 From: Zia <194475824+electricEpilith@users.noreply.github.com> Date: Wed, 8 Apr 2026 08:32:08 -0700 Subject: [PATCH 75/77] fix duplicate declaration bug Co-Authored-By: Claude Sonnet 4.6 --- Makefile | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index 43531d6ff5..e3764b0291 100644 --- a/Makefile +++ b/Makefile @@ -375,7 +375,7 @@ BBHASH_DIR=deps/BBHash MIO_DIR=deps/mio ATOMIC_QUEUE_DIR=deps/atomic_queue THESEUS_DIR:=deps/theseus-lib -INCLUDE_FLAGS += -I$(CWD)/$(THESEUS_DIR) +INCLUDE_FLAGS += -I$(CWD)/$(THESEUS_DIR)/include # Dependencies that go into libvg's archive # These go in libvg but come from dependencies @@ -940,8 +940,7 @@ $(LIB_DIR)/libxg.a: $(XG_DIR)/src/*.hpp $(XG_DIR)/src/*.cpp $(INC_DIR)/mmmultima +ar rs $@ $(XG_DIR)/xg.o $(LIB_DIR)/libtheseus.a: $(LIB_DIR)/libhandlegraph.a $(wildcard $(THESEUS_DIR)/theseus/*.cpp) $(wildcard $(THESEUS_DIR)/include/theseus/*.h) - +rm -Rf $(CWD)/$(INC_DIR)/theseus - +cd $(THESEUS_DIR) && rm -Rf build && mkdir build && cd build && cmake -DCMAKE_C_COMPILER="$(CC)" -DCMAKE_CXX_COMPILER="$(CXX)" -DCMAKE_VERBOSE_MAKEFILE=ON -DCMAKE_C_FLAGS="-fPIC $(CFLAGS)" -DCMAKE_CXX_FLAGS="-fPIC $(CPPFLAGS)" -DCMAKE_BUILD_TYPE=Release -DENABLE_TESTS=OFF -DFETCHCONTENT_SOURCE_DIR_LIBHANDLEGRAPH=$(CWD)/$(LIBHANDLEGRAPH_DIR) .. $(FILTER) && $(MAKE) theseus $(FILTER) && cp libtheseus.a $(CWD)/$(LIB_DIR)/ && cp -r $(CWD)/$(THESEUS_DIR)/include/theseus $(CWD)/$(INC_DIR)/ + +cd $(THESEUS_DIR) && rm -Rf build && mkdir build && cd build && cmake -DCMAKE_C_COMPILER="$(CC)" -DCMAKE_CXX_COMPILER="$(CXX)" -DCMAKE_VERBOSE_MAKEFILE=ON -DCMAKE_C_FLAGS="-fPIC $(CFLAGS)" -DCMAKE_CXX_FLAGS="-fPIC $(CPPFLAGS)" -DCMAKE_BUILD_TYPE=Release -DENABLE_TESTS=OFF -DFETCHCONTENT_SOURCE_DIR_LIBHANDLEGRAPH=$(CWD)/$(LIBHANDLEGRAPH_DIR) .. $(FILTER) && $(MAKE) theseus $(FILTER) && cp libtheseus.a $(CWD)/$(LIB_DIR)/ # Auto-git-versioning From 9186ebcb7cfa85aa50a0d40add508d99b81e9d8b Mon Sep 17 00:00:00 2001 From: Zia <194475824+electricEpilith@users.noreply.github.com> Date: Wed, 8 Apr 2026 10:03:50 -0700 Subject: [PATCH 76/77] add stuff --- src/minimizer_mapper_from_chains.cpp | 45 +++++++++++++++++++++++----- src/theseus_interop.hpp | 4 ++- 2 files changed, 41 insertions(+), 8 deletions(-) diff --git a/src/minimizer_mapper_from_chains.cpp b/src/minimizer_mapper_from_chains.cpp index ac2cd0c138..11a43facfd 100644 --- a/src/minimizer_mapper_from_chains.cpp +++ b/src/minimizer_mapper_from_chains.cpp @@ -3118,8 +3118,8 @@ Alignment MinimizerMapper::find_chain_alignment( return result; } -Alignment MinimizerMapper::find_chain_alignment_theseus( - const Alignment& aln, +vg::Alignment MinimizerMapper::find_chain_alignment_theseus( + const vg::Alignment& aln, const VectorView& to_chain, const std::vector& chain, aligner_stats_t* stats @@ -3128,14 +3128,45 @@ Alignment MinimizerMapper::find_chain_alignment_theseus( if (chain.empty()) { throw ChainAlignmentFailedError("Cannot find an alignment for an empty chain!"); } - - //bdsg::HashGraph local_graph = get_local_graph(); - - + //confusingly only needed for getting penalty values + const Aligner& aligner = *get_regular_aligner(); + + string query_seq = aln.sequence(); + string_view query_view(query_seq); + HandleGraphTheseusAdapter theseus_adapter(this->gbwt_graph); + theseus::Penalties theseus_penalties( + aligner.match, + aligner.mismatch, + aligner.gap_open, + aligner.gap_extension + ); + + bool is_msa = false; + theseus::TheseusAlignerImpl actual_aligner( + theseus_penalties, + theseus_adapter.take_graph(), //only call take_graph() once, since it moves the graph out of the adapter + is_msa + ); + + const algorithms::Anchor start_anchor = to_chain[chain.front()]; + const pos_t start_pos = start_anchor.graph_start(); + const vg::id_t start_id = id(start_pos); + const handle_t start_handle = this->gbwt_graph.get_handle(start_id); + string start_handle_name = theseus_adapter.handle_name(this->gbwt_graph, start_handle); + theseus::Alignment theseus_alignment = actual_aligner.align( + query_view, + start_handle_name, + start_anchor.start_hint_offset() + ); // Convert to a vg Alignment. - Alignment result; + vg::Alignment result = vg_alignment_from_theseus_alignment( + theseus_alignment, + query_seq, + theseus_adapter, + this->gbwt_graph + ); /* // Simplify the path but keep internal deletions; we want to assert the // read deleted relative to some graph, and avoid jumps along nonexistent diff --git a/src/theseus_interop.hpp b/src/theseus_interop.hpp index e8fa5e14b7..9e62086afd 100644 --- a/src/theseus_interop.hpp +++ b/src/theseus_interop.hpp @@ -6,13 +6,15 @@ #include #include -#include +#include #include /** * @file theseus_interop.hpp * * @brief Interop for using Theseus in vg + * + * [PROVISIONAL] This file is mostly placeholder code */ namespace vg { From bbd96ea39cd55a0ba9b42bbba1a819fa6d599c3c Mon Sep 17 00:00:00 2001 From: Zia <194475824+electricEpilith@users.noreply.github.com> Date: Wed, 8 Apr 2026 10:07:29 -0700 Subject: [PATCH 77/77] commit theseus-lib changes --- deps/theseus-lib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deps/theseus-lib b/deps/theseus-lib index 05277907e5..64b00118f1 160000 --- a/deps/theseus-lib +++ b/deps/theseus-lib @@ -1 +1 @@ -Subproject commit 05277907e5a5901002699eb43dfb4efef4f3acf9 +Subproject commit 64b00118f11c510bf665bbe10fcadfc3e6494750