def main()

in batch/aiml-workloads/src/worker.py [0:0]

• 23 lines of code
• 4 McCabe index (conditional complexity)

def main():
    """
    Workload which:
      1. Claims a filename from a Redis Worker Queue
      2. Reads the dataset from the file
      3. Partially trains the model on the dataset
      4. Saves a model checkpoint and generates a report on
         the performance of the model after the partial training.
      5. Removes the filename from the Redis Worker Queue
      6. Repeats 1 through 5 till the Queue is empty
    """
    q = rediswq.RedisWQ(name="datasets", host=HOST)
    print("Worker with sessionID: " + q.sessionID())
    print("Initial queue state: empty=" + str(q.empty()))
    checkpoint_path = None
    while not q.empty():
        # Claim item in Redis Worker Queue
        item = q.lease(lease_secs=20, block=True, timeout=2)
        if item is not None:
            dataset_path = item.decode("utf-8")
            print("Processing dataset: " + dataset_path)
            training_dataset_path = FILESTORE_PATH + dataset_path

            # Initialize the model training manager class
            model_trainer = FraudDetectionModelTrainer(
                training_dataset_path,
                TESTING_DATASET_PATH,
                CLASS_LABEL,
                checkpoint_path=checkpoint_path,
            )

            # Train model and save checkpoint + report
            checkpoint_path = model_trainer.train_and_save(OUTPUT_DIR)
            model_trainer.generate_report(REPORT_PATH)

            # Remove item from Redis Worker Queue
            q.complete(item)
        else:
            print("Waiting for work")

    print("Queue empty, exiting")