in clns-acuity-admin/acuity-etl/src/main/java/com/acuity/visualisations/batch/processor/InputModelChunkProcessorImpl.java [290:466]
private List<OutputEntity> createOutputEntity(String sourceName, String entityName, Class<?> entityClass, String[] record,
Set<String> columnsToSkip, int rowNumber, int offSet) throws Exception {
executionProfiler.startOperation(getJobExecutionId(), "InputModelChunkProcessorImpl.createOutputEntity");
class CompoundFields {
private List<IColumnRule> columnRules = new ArrayList<>();
private List<Object> compoundValues = new ArrayList<>();
void addEntry(IColumnRule columnRule, Object compoundValue) {
columnRules.add(columnRule);
compoundValues.add(compoundValue);
}
}
Map<String, Pair<String, Object>> pivotedValues = new HashMap<>();
OutputEntity entity = (OutputEntity) entityClass.newInstance();
Map<String, CompoundFields> compoundFieldsParts = new HashMap<>();
int missedColumnsOffset = 0;
List<? extends IColumnRule> columns = configurationUtil.getColumns(sourceName, entityName);
if (columns == null) {
return null;
}
for (int i = offSet; i < columns.size() + offSet; i++) {
IColumnRule columnRule = columns.get(i - offSet);
if (columnsToSkip.contains(columnRule.getName())) {
missedColumnsOffset++;
continue;
}
int recordIndex = i - missedColumnsOffset;
executionProfiler.startOperation(getJobExecutionId(), "InputModelChunkProcessorImpl.createOutputEntity-int1");
String recValue = recordIndex >= 0 && recordIndex < record.length ? record[recordIndex] : null;
for (String value : columnRule.getExcludingValues()) {
if ((recValue != null && value != null
&& value.toUpperCase().equals(recValue.toUpperCase()))
|| ("NULL".equalsIgnoreCase(value) && (isEmpty(recValue)))) {
excludedRows.add(rowNumber);
}
}
entity.addSourceColumn(columnRule.getName(), recValue);
executionProfiler.stopOperation(getJobExecutionId(), "InputModelChunkProcessorImpl.createOutputEntity-int1");
executionProfiler.startOperation(getJobExecutionId(), "InputModelChunkProcessorImpl.createOutputEntity-int2");
Object merged = mergeWithCntlin(columnRule, recValue, sourceName, rowNumber, true);
if (merged == null || "NULL".equalsIgnoreCase((String) merged)) {
recValue = null;
} else {
recValue = (String) merged;
}
executionProfiler.stopOperation(getJobExecutionId(), "InputModelChunkProcessorImpl.createOutputEntity-int2");
executionProfiler.startOperation(getJobExecutionId(), "InputModelChunkProcessorImpl.createOutputEntity-int3");
Object value = null;
String type = columnRule.getType();
Mapper mapper = columnRule.getMapper();
ParserRule helper = columnRule.getHelper();
executionProfiler.stopOperation(getJobExecutionId(), "InputModelChunkProcessorImpl.createOutputEntity-int3");
executionProfiler.startOperation(getJobExecutionId(), "InputModelChunkProcessorImpl.createOutputEntity-report");
if (columnRule.getSubjectField()) {
dataCommonReport.getFileReport(sourceName).addParsedSubject(recValue);
}
dataCommonReport.getFileReport(sourceName).incColumnParsed(columnRule.getName());
executionProfiler.stopOperation(getJobExecutionId(), "InputModelChunkProcessorImpl.createOutputEntity-report");
executionProfiler.stopOperation(getJobExecutionId(), "InputModelChunkProcessorImpl.createOutputEntity-int7");
// try to use mapper function
String functionName = columnRule.getAggrFunction();
AbstractParser<?> parser = ReflectionUtil.getParser(getJobExecutionId(), sourceName, columnRule.getName(), type, mapper, helper);
try {
value = parser.parse(recValue);
} catch (InvalidDataFormatException e) {
dataCommonReport.getFileReport(sourceName).addValueError(columnRule.getName(), recValue, true,
"Error parsing value of expected type \"" + type + "\": " + e.getMessage());
}
if (!isTrue(columnRule.isPart())) {
Object value1;
if (value instanceof Temporal) {
if (value instanceof LocalDate) {
value1 = Date.from(((LocalDate) value).atStartOfDay().atZone(ZoneId.systemDefault()).toInstant());
} else {
if (value instanceof LocalDateTime) {
value1 = Date.from(((LocalDateTime) value).atZone(ZoneId.systemDefault()).toInstant());
} else {
value1 = null;
}
}
} else {
value1 = value;
}
setFieldValue(entity, columnRule, value1, false, pivotedValues);
} else {
if (!compoundFieldsParts.containsKey(columnRule.getField())) {
compoundFieldsParts.put(columnRule.getField(), new CompoundFields());
}
compoundFieldsParts.get(columnRule.getField()).addEntry(columnRule, value);
}
if (functionName != null) {
if (compoundFieldsParts.get(columnRule.getField()) != null) {
AbstractFunction aggregatorFunction = Functions.getAggregator(functionName);
List<Object> compoundValues = compoundFieldsParts.get(columnRule.getField()).compoundValues;
if (aggregatorFunction instanceof DateAssemblerDefaultTime && !isEmpty(columnRule.getDefault())) {
Object defTime = null;
try {
defTime = parser.parse(columnRule.getDefault());
} catch (InvalidDataFormatException ignored) {
}
compoundValues.add(defTime);
}
Object[] params = compoundValues.toArray();
Object functionedValue;
if (aggregatorFunction != null) {
functionedValue = aggregatorFunction.function(params);
functionedValue = mergeWithCntlinArray(columnRule, functionedValue, sourceName, rowNumber, false);
setFieldValue(entity, columnRule, functionedValue, true, pivotedValues);
compoundFieldsParts.remove(columnRule.getField());
}
} else {
executionProfiler.startOperation(getJobExecutionId(), "InputModelChunkProcessorImpl.setFieldValue-2");
setFieldValue(entity, columnRule, null, false, pivotedValues);
executionProfiler.stopOperation(getJobExecutionId(), "InputModelChunkProcessorImpl.setFieldValue-2");
}
}
executionProfiler.stopOperation(getJobExecutionId(), "InputModelChunkProcessorImpl.createOutputEntity-int7");
}
for (Map.Entry<String, CompoundFields> entry : compoundFieldsParts.entrySet()) {
CompoundFields fields = entry.getValue();
IColumnRule columnRule = fields.columnRules.get(0);
Object value = fields.compoundValues.get(0);
value = mergeWithCntlin(columnRule, value, sourceName, rowNumber, false);
setFieldValue(entity, columnRule, value, false, pivotedValues);
debug("Unnecessary part attribute for Field: {}, Entity: {}, Study: {}, Project: {}. ", columnRule.getField(),
entityName, getStudyName(), getProjectName());
}
executionProfiler.stopOperation(getJobExecutionId(), "InputModelChunkProcessorImpl.createOutputEntity");
List<OutputEntity> outputEntities = new ArrayList<>(1);
if (entity instanceof PivotableEntity && !pivotedValues.isEmpty()) {
pivotedValues.entrySet().stream().forEach(entry -> {
try {
PivotableEntity pivotedEntity = (PivotableEntity) entity.clone();
pivotedEntity.setPivotedCategoryValue(entry.getValue().getA(), entry.getKey(), entry.getValue().getB());
outputEntities.add((OutputEntity) pivotedEntity);
} catch (Exception e) {
error("Exception caught when assigning pivoted entity value: {}", e.getMessage());
LOGGER.error("Exception caught when assigning pivoted entity value: ", e);
}
});
} else if (entity instanceof SplitEntity) {
if (entity instanceof SmartEntity) {
((SmartEntity) entity).complete();
}
outputEntities.addAll(((SplitEntity) entity).split());
} else {
outputEntities.add(entity);
}
for (OutputEntity outputEntity : outputEntities) {
if (outputEntity instanceof SmartEntity) {
((SmartEntity) outputEntity).complete();
}
}
return outputEntities;
}