in cassovary-core/src/main/scala/com/twitter/cassovary/graph/ArrayBasedDirectedGraph.scala [170:195]
private def fillOutEdges(iterableSeq: Seq[Iterable[NodeIdEdgesMaxId]]):
Future[GraphInfo[Seq[Node]]] = {
log.debug("loading nodes and out edges from file in parallel")
val nodesOutEdges = new mutable.ArrayBuffer[Seq[Node]]
var maxNodeIdAll = 0
var numEdgesAll = 0L
var numNodesAll = 0
val outEdgesAll: Future[Seq[GraphInfo[Node]]] = Stat.time(
statsReceiver.stat("graph_dump_load_partial_nodes_and_out_edges_parallel")) {
Future.collect(iterableSeq.map(i => readOutEdges(i.iterator)))
}
outEdgesAll.map {
// aggregate across parts
case outEdgesOnePart => outEdgesOnePart.foreach {
case GraphInfo(nodesInPart, maxIdInPart, numNodesInPart, numEdgesInPart) =>
nodesOutEdges += nodesInPart
maxNodeIdAll = maxNodeIdAll max maxIdInPart
numNodesAll += numNodesInPart
numEdgesAll += numEdgesInPart
}
GraphInfo[Seq[Node]](nodesOutEdges, maxNodeId = maxNodeIdAll,
numNodes = numNodesAll, numEdges = numEdgesAll)
}
}