in sdk/src/main/java/com/google/cloud/dataflow/sdk/runners/DataflowPipelineTranslator.java [391:454]
public Job translate(List<DataflowPackage> packages) {
job.setName(options.getJobName().toLowerCase());
Environment environment = new Environment();
job.setEnvironment(environment);
try {
environment.setSdkPipelineOptions(
MAPPER.readValue(MAPPER.writeValueAsBytes(options), Map.class));
} catch (IOException e) {
throw new IllegalArgumentException(
"PipelineOptions specified failed to serialize to JSON.", e);
}
WorkerPool workerPool = new WorkerPool();
if (options.getTeardownPolicy() != null) {
workerPool.setTeardownPolicy(options.getTeardownPolicy().getTeardownPolicyName());
}
if (options.isStreaming()) {
job.setType("JOB_TYPE_STREAMING");
} else {
job.setType("JOB_TYPE_BATCH");
workerPool.setDiskType(options.getWorkerDiskType());
}
if (options.getWorkerMachineType() != null) {
workerPool.setMachineType(options.getWorkerMachineType());
}
workerPool.setPackages(packages);
workerPool.setNumWorkers(options.getNumWorkers());
if (options.isStreaming()) {
// Use separate data disk for streaming.
Disk disk = new Disk();
disk.setDiskType(options.getWorkerDiskType());
workerPool.setDataDisks(Collections.singletonList(disk));
}
if (!Strings.isNullOrEmpty(options.getZone())) {
workerPool.setZone(options.getZone());
}
if (!Strings.isNullOrEmpty(options.getNetwork())) {
workerPool.setNetwork(options.getNetwork());
}
if (options.getDiskSizeGb() > 0) {
workerPool.setDiskSizeGb(options.getDiskSizeGb());
}
if (options.getAutoscalingAlgorithm() != null) {
AutoscalingSettings settings = new AutoscalingSettings();
settings.setAlgorithm(options.getAutoscalingAlgorithm().getAlgorithm());
settings.setMaxNumWorkers(options.getMaxNumWorkers());
workerPool.setAutoscalingSettings(settings);
}
List<WorkerPool> workerPools = new LinkedList<>();
workerPools.add(workerPool);
environment.setWorkerPools(workerPools);
pipeline.traverseTopologically(this);
return job;
}