def run()

in spotify_tensorflow/tfx/tft.py [0:0]

• 43 lines of code
• 3 McCabe index (conditional complexity)

    def run(self, args=None):
        parser = argparse.ArgumentParser()
        parser.add_argument(
            "--training_data",
            required=False,
            help="path to the raw feature for training")
        parser.add_argument(
            "--evaluation_data",
            required=False,
            help="path to the raw feature for evaluation")
        parser.add_argument(
            "--output_dir",
            required=True,
            help="output dir for data transformation")
        parser.add_argument(
            "--schema_file",
            required=True,
            help="path to the schema txt file")
        parser.add_argument(
            "--temp_location",
            required=True,
            help="temporary working dir for tf.transform job")
        parser.add_argument(
            "--transform_fn_dir",
            required=False,
            help="path to the saved transform function")
        parser.add_argument(
            "--compression_type",
            required=False,
            help="compression type for writing of tf.records")

        if args is None:
            args = sys.argv[1:]
        tft_args, pipeline_args = parser.parse_known_args(args=args)

        # pipeline_args also needs temp_location and requirements_file
        pipeline_args.append("--temp_location=%s" % tft_args.temp_location)

        tftransform(pipeline_args=pipeline_args,
                    temp_location=tft_args.temp_location,
                    schema_file=tft_args.schema_file,
                    output_dir=tft_args.output_dir,
                    preprocessing_fn=self.preprocessing_fn,
                    training_data=tft_args.training_data,
                    evaluation_data=tft_args.evaluation_data,
                    transform_fn_dir=tft_args.transform_fn_dir,
                    compression_type=tft_args.compression_type)