in src/main/java/com/hadoop/compression/lzo/LzoIndexer.java [77:107]
private void indexInternal(Path lzoPath, int nestingLevel) throws IOException {
FileSystem fs = FileSystem.get(URI.create(lzoPath.toString()), conf_);
FileStatus fileStatus = fs.getFileStatus(lzoPath);
// Recursively walk
if (fileStatus.isDir()) {
LOG.info(getNesting(nestingLevel) + "LZO Indexing directory " + lzoPath + "...");
FileStatus[] statuses = fs.listStatus(lzoPath);
for (FileStatus childStatus: statuses) {
indexInternal(childStatus.getPath(), nestingLevel + 1);
}
} else if (lzoPath.toString().endsWith(LZO_EXTENSION)) {
Path lzoIndexPath = new Path(lzoPath.toString() + LzoIndex.LZO_INDEX_SUFFIX);
if (fs.exists(lzoIndexPath)) {
LOG.info(getNesting(nestingLevel) + "[SKIP] LZO index file already exists for " + lzoPath + "\n");
} else {
long startTime = System.currentTimeMillis();
long fileSize = fileStatus.getLen();
LOG.info(getNesting(nestingLevel) + "[INDEX] LZO Indexing file " + lzoPath + ", size " +
df_.format(fileSize / (1024.0 * 1024.0 * 1024.0)) + " GB...");
if (indexSingleFile(fs, lzoPath)) {
long indexSize = fs.getFileStatus(lzoIndexPath).getLen();
double elapsed = (System.currentTimeMillis() - startTime) / 1000.0;
LOG.info(getNesting(nestingLevel) + "Completed LZO Indexing in " + df_.format(elapsed) + " seconds (" +
df_.format(fileSize / (1024.0 * 1024.0 * elapsed)) + " MB/s). Index size is " +
df_.format(indexSize / 1024.0) + " KB.\n");
}
}
}
}