in cassovary-core/src/main/scala/com/twitter/cassovary/graph/SharedArrayBasedDirectedGraph.scala [82:106]
private def readMetaInfoPerShard():
Future[Seq[SharedGraphMetaInfo]] = {
log.debug("read out num of edges and max id from files in parallel")
val stat = statsReceiver.stat("graph_load_read_out_edge_sizes_dump_files")
Stat.timeFuture(stat) {
val futures = iterableSeq map {
edgesIterable => futurePool {
var id, newMaxId, numOfEdges, edgesLength, numNodes, shardNum = 0
val iteratorForEdgeSizes = edgesIterable.iterator
iteratorForEdgeSizes foreach { item =>
id = item.id
newMaxId = newMaxId max item.maxId
edgesLength = item.edges.length
shardNum = EdgeShards.hash(id)
// +1 because we will keep edgesLength in the very first index
shardsInfo(shardNum).numEdges.addAndGet(edgesLength + 1)
numOfEdges += edgesLength
numNodes += 1
}
SharedGraphMetaInfo(newMaxId, numOfEdges, numNodes)
}
}
Future.collect(futures)
}
}