in parquet/src/main/scala/magnolify/parquet/Schema.scala [98:153]
def checkCompatibility(writer: Type, reader: Type): Unit = {
def listFields(gt: GroupType) =
s"[${gt.getFields.asScala.map(f => s"${f.getName}: ${f.getRepetition}").mkString(",")}]"
def isRepetitionBackwardCompatible(w: Type, r: Type) =
(w.getRepetition, r.getRepetition) match {
case (Repetition.REQUIRED, Repetition.OPTIONAL) => true
case (r1, r2) => r1 == r2
}
if (
!isRepetitionBackwardCompatible(writer, reader) ||
writer.isPrimitive != reader.isPrimitive
) {
throw new InvalidRecordException(s"$writer found: expected $reader")
}
writer match {
case wg: GroupType =>
val rg = reader.asGroupType()
rg.getFields.asScala.foreach { rf =>
if (wg.containsField(rf.getName)) {
val wf = wg.getType(rf.getName)
checkCompatibility(wf, rf)
} else {
(
rf.getLogicalTypeAnnotation != LogicalTypeAnnotation.listType(),
rf.getRepetition
) match {
case (true, Repetition.REQUIRED) =>
throw new InvalidRecordException(
s"Requested field `${rf.getName}: ${rf.getRepetition}` is not present in written file schema. " +
s"Available fields are: ${listFields(wg)}"
)
case (true, Repetition.OPTIONAL) =>
logger.warn(
s"Requested field `${rf.getName}: ${rf.getRepetition}` is not present in written file schema " +
s"and will be evaluated as `Option.empty`. Available fields are: ${listFields(wg)}"
)
case _ =>
}
}
}
case _: PrimitiveType =>
val wf = writer.asPrimitiveType()
val rf = reader.asPrimitiveType()
if (wf.getPrimitiveTypeName != rf.getPrimitiveTypeName) {
throw new InvalidRecordException(
s"Requested ${reader.getName} with primitive type $rf not " +
s"found; written file schema had type $wf"
)
}
case _ =>
throw new Exception(s"Unsupported type for $writer")
}
}