Working on eval-script for Keras

eval_ke.py

@@ -2,11 +2,11 @@
 import time
 import logging
 
-import mxnet as mx
+import keras
+from keras.utils.np_utils import to_categorical
 
 from common.logger_utils import initialize_logging
-from gluon.utils import prepare_mx_context, prepare_model, get_data_rec, get_data_loader, calc_net_weight_count,\
-    validate
+from keras_.utils import prepare_ke_context, prepare_model, get_data_rec, backend_agnostic_compile
 
 
 def parse_args():
@@ -86,49 +86,102 @@ def parse_args():
     parser.add_argument(
         '--log-packages',
         type=str,
-        default='mxnet',
+        default='keras',
         help='list of python packages for logging')
     parser.add_argument(
         '--log-pip-packages',
         type=str,
-        default='mxnet-cu92, gluoncv',
+        default='keras, keras-mxnet, keras-applications, keras-preprocessing',
         help='list of pip packages for logging')
     args = parser.parse_args()
     return args
 
 
+def get_data(it,
+             batch_size,
+             num_classes,
+             report_speed=False,
+             warm_batches_up_for_reporting=100):
+    ctr = 0
+    warm_up_done = False
+    last_time = None
+
+    # Need to feed data as NumPy arrays to Keras
+    def get_arrays(db):
+        return db.data[0].asnumpy().transpose((0, 2, 3, 1)),\
+               to_categorical(
+                   y=db.label[0].asnumpy(),
+                   num_classes=num_classes)
+
+    # repeat for as long as training is to proceed, reset iterator if need be
+    while True:
+        try:
+            ctr += 1
+            db = it.next()
+
+            # Skip this if samples/second reporting is not desired
+            if report_speed:
+
+                # Report only after warm-up is done to prevent downward bias
+                if warm_up_done:
+                    curr_time = time()
+                    elapsed = curr_time - last_time
+                    ss = float(batch_size * ctr) / elapsed
+                    print(" Batch: %d, Samples per sec: %.2f" % (ctr, ss))
+
+                if ctr > warm_batches_up_for_reporting and not warm_up_done:
+                    ctr = 0
+                    last_time = time()
+                    warm_up_done = True
+
+        except StopIteration as e:
+            print("get_data exception due to end of data - resetting iterator")
+            it.reset()
+            db = it.next()
+
+        finally:
+            yield get_arrays(db)
+
+
 def test(net,
-         val_data,
-         batch_fn,
-         use_rec,
-         dtype,
-         ctx,
+         val_gen,
+         val_size,
+         batch_size,
+         num_gpus,
          calc_weight_count=False,
          extended_log=False):
-    acc_top1 = mx.metric.Accuracy()
-    acc_top5 = mx.metric.TopKAccuracy(5)
 
+    backend_agnostic_compile(
+        model=net,
+        loss='categorical_crossentropy',
+        optimizer=keras.optimizers.SGD(
+            lr=0.01,
+            momentum=0.0,
+            decay=0.0,
+            nesterov=False),
+        metrics=['accuracy'],
+        num_gpus=num_gpus)
+
+    #net.summary()
     tic = time.time()
-    err_top1_val, err_top5_val = validate(
-        acc_top1=acc_top1,
-        acc_top5=acc_top5,
-        net=net,
-        val_data=val_data,
-        batch_fn=batch_fn,
-        use_rec=use_rec,
-        dtype=dtype,
-        ctx=ctx)
+    score = net.evaluate_generator(
+        generator=val_gen,
+        steps=(val_size // batch_size),
+        verbose=True)
     if calc_weight_count:
-        weight_count = calc_net_weight_count(net)
+        weight_count = keras.utils.layer_utils.count_params(net.trainable_weights)
         logging.info('Model: {} trainable parameters'.format(weight_count))
-    if extended_log:
-        logging.info('Test: err-top1={top1:.4f} ({top1})\terr-top5={top5:.4f} ({top5})'.format(
-            top1=err_top1_val, top5=err_top5_val))
-    else:
-        logging.info('Test: err-top1={top1:.4f}\terr-top5={top5:.4f}'.format(
-            top1=err_top1_val, top5=err_top5_val))
+    # if extended_log:
+    #     logging.info('Test: err-top1={top1:.4f} ({top1})\terr-top5={top5:.4f} ({top5})'.format(
+    #         top1=err_top1_val, top5=err_top5_val))
+    # else:
+    #     logging.info('Test: err-top1={top1:.4f}\terr-top5={top5:.4f}'.format(
+    #         top1=err_top1_val, top5=err_top5_val))
     logging.info('Time cost: {:.4f} sec'.format(
         time.time() - tic))
+    logging.info('score: {}'.format(score))
+    logging.info('Test score: {}'.format(score[0]))
+    logging.info('Test accuracy: {}'.format(score[1]))
 
 
 def main():
@@ -141,7 +194,7 @@ def main():
         log_packages=args.log_packages,
         log_pip_packages=args.log_pip_packages)
 
-    ctx, batch_size = prepare_mx_context(
+    batch_size = prepare_ke_context(
         num_gpus=args.num_gpus,
         batch_size=args.batch_size)
 
@@ -150,34 +203,35 @@ def main():
         model_name=args.model,
         classes=num_classes,
         use_pretrained=args.use_pretrained,
-        pretrained_model_file_path=args.resume.strip(),
-        dtype=args.dtype,
-        tune_layers="",
-        ctx=ctx)
-
-    if args.use_rec:
-        train_data, val_data, batch_fn = get_data_rec(
-            rec_train=args.rec_train,
-            rec_train_idx=args.rec_train_idx,
-            rec_val=args.rec_val,
-            rec_val_idx=args.rec_val_idx,
-            batch_size=batch_size,
-            num_workers=args.num_workers)
-    else:
-        train_data, val_data, batch_fn = get_data_loader(
-            data_dir=args.data_dir,
-            batch_size=batch_size,
-            num_workers=args.num_workers)
+        pretrained_model_file_path=args.resume.strip())
+
+    train_data, val_data = get_data_rec(
+        rec_train=args.rec_train,
+        rec_train_idx=args.rec_train_idx,
+        rec_val=args.rec_val,
+        rec_val_idx=args.rec_val_idx,
+        batch_size=batch_size,
+        num_workers=args.num_workers)
+
+    # train_gen = get_data(
+    #     it=train_data,
+    #     batch_size=batch_size,
+    #     num_classes=num_classes,
+    #     report_speed=True)
+    val_gen = get_data(
+        it=val_data,
+        batch_size=batch_size,
+        num_classes=num_classes,
+        report_speed=True)
+    val_size = 50000
 
     assert (args.use_pretrained or args.resume.strip())
     test(
         net=net,
-        val_data=val_data,
-        batch_fn=batch_fn,
-        use_rec=args.use_rec,
-        dtype=args.dtype,
-        ctx=ctx,
-        # calc_weight_count=(not log_file_exist),
+        val_gen=val_gen,
+        val_size=val_size,
+        batch_size=batch_size,
+        num_gpus=args.num_gpus,
         calc_weight_count=True,
         extended_log=True)
 

keras_/utils.py

@@ -1,14 +1,77 @@
 import logging
 import os
-import numpy as np
 
-import torch.utils.data
-import torchvision.transforms as transforms
-import torchvision.datasets as datasets
+import keras
+import mxnet as mx
 
 from .model_provider import get_model
 
 
+def prepare_ke_context(num_gpus,
+                       batch_size):
+    batch_size *= max(1, num_gpus)
+    return batch_size
+
+
+def get_data_rec(rec_train,
+                 rec_train_idx,
+                 rec_val,
+                 rec_val_idx,
+                 batch_size,
+                 num_workers):
+    rec_train = os.path.expanduser(rec_train)
+    rec_train_idx = os.path.expanduser(rec_train_idx)
+    rec_val = os.path.expanduser(rec_val)
+    rec_val_idx = os.path.expanduser(rec_val_idx)
+    jitter_param = 0.4
+    lighting_param = 0.1
+    mean_rgb = [123.68, 116.779, 103.939]
+    std_rgb = [58.393, 57.12, 57.375]
+
+    train_data = mx.io.ImageRecordIter(
+        path_imgrec=rec_train,
+        path_imgidx=rec_train_idx,
+        preprocess_threads=num_workers,
+        shuffle=True,
+        batch_size=batch_size,
+
+        data_shape=(3, 224, 224),
+        mean_r=mean_rgb[0],
+        mean_g=mean_rgb[1],
+        mean_b=mean_rgb[2],
+        std_r=std_rgb[0],
+        std_g=std_rgb[1],
+        std_b=std_rgb[2],
+        rand_mirror=True,
+        random_resized_crop=True,
+        max_aspect_ratio=(4. / 3.),
+        min_aspect_ratio=(3. / 4.),
+        max_random_area=1,
+        min_random_area=0.08,
+        brightness=jitter_param,
+        saturation=jitter_param,
+        contrast=jitter_param,
+        pca_noise=lighting_param,
+    )
+    val_data = mx.io.ImageRecordIter(
+        path_imgrec=rec_val,
+        path_imgidx=rec_val_idx,
+        preprocess_threads=num_workers,
+        shuffle=False,
+        batch_size=batch_size,
+
+        resize=256,
+        data_shape=(3, 224, 224),
+        mean_r=mean_rgb[0],
+        mean_g=mean_rgb[1],
+        mean_b=mean_rgb[2],
+        std_r=std_rgb[0],
+        std_g=std_rgb[1],
+        std_b=std_rgb[2],
+    )
+    return train_data, val_data
+
+
 def prepare_model(model_name,
                   classes,
                   use_pretrained,
@@ -22,8 +85,26 @@ def prepare_model(model_name,
         assert (os.path.isfile(pretrained_model_file_path))
         logging.info('Loading model: {}'.format(pretrained_model_file_path))
         net.load_weights(filepath=pretrained_model_file_path)
-        #from keras.models import load_model
-        #net = load_model(pretrained_model_file_path)
-
 
     return net
+
+
+def backend_agnostic_compile(model,
+                             loss,
+                             optimizer,
+                             metrics,
+                             num_gpus):
+    if keras.backend._backend == 'mxnet':
+        gpu_list = ["gpu(%d)" % i for i in range(num_gpus)]
+        model.compile(
+            loss=loss,
+            optimizer=optimizer,
+            metrics=metrics,
+            context=gpu_list)
+    else:
+        if num_gpus > 1:
+            print("Warning: num_gpus > 1 but not using MxNet backend")
+        model.compile(
+            loss=loss,
+            optimizer=optimizer,
+            metrics=metrics)