From ef4170e3336f2192ebb177e029804c9847f8c716 Mon Sep 17 00:00:00 2001 From: osmr Date: Wed, 19 Sep 2018 19:51:06 +0300 Subject: [PATCH] Working on eval-script for Keras --- eval_ke.py | 160 ++++++++++++++++++++++++++++++++---------------- keras_/utils.py | 95 +++++++++++++++++++++++++--- 2 files changed, 195 insertions(+), 60 deletions(-) diff --git a/eval_ke.py b/eval_ke.py index 78d12706a..7b137c725 100644 --- a/eval_ke.py +++ b/eval_ke.py @@ -2,11 +2,11 @@ import time import logging -import mxnet as mx +import keras +from keras.utils.np_utils import to_categorical from common.logger_utils import initialize_logging -from gluon.utils import prepare_mx_context, prepare_model, get_data_rec, get_data_loader, calc_net_weight_count,\ - validate +from keras_.utils import prepare_ke_context, prepare_model, get_data_rec, backend_agnostic_compile def parse_args(): @@ -86,49 +86,102 @@ def parse_args(): parser.add_argument( '--log-packages', type=str, - default='mxnet', + default='keras', help='list of python packages for logging') parser.add_argument( '--log-pip-packages', type=str, - default='mxnet-cu92, gluoncv', + default='keras, keras-mxnet, keras-applications, keras-preprocessing', help='list of pip packages for logging') args = parser.parse_args() return args +def get_data(it, + batch_size, + num_classes, + report_speed=False, + warm_batches_up_for_reporting=100): + ctr = 0 + warm_up_done = False + last_time = None + + # Need to feed data as NumPy arrays to Keras + def get_arrays(db): + return db.data[0].asnumpy().transpose((0, 2, 3, 1)),\ + to_categorical( + y=db.label[0].asnumpy(), + num_classes=num_classes) + + # repeat for as long as training is to proceed, reset iterator if need be + while True: + try: + ctr += 1 + db = it.next() + + # Skip this if samples/second reporting is not desired + if report_speed: + + # Report only after warm-up is done to prevent downward bias + if warm_up_done: + curr_time = time() + elapsed = curr_time - last_time + ss = float(batch_size * ctr) / elapsed + print(" Batch: %d, Samples per sec: %.2f" % (ctr, ss)) + + if ctr > warm_batches_up_for_reporting and not warm_up_done: + ctr = 0 + last_time = time() + warm_up_done = True + + except StopIteration as e: + print("get_data exception due to end of data - resetting iterator") + it.reset() + db = it.next() + + finally: + yield get_arrays(db) + + def test(net, - val_data, - batch_fn, - use_rec, - dtype, - ctx, + val_gen, + val_size, + batch_size, + num_gpus, calc_weight_count=False, extended_log=False): - acc_top1 = mx.metric.Accuracy() - acc_top5 = mx.metric.TopKAccuracy(5) + backend_agnostic_compile( + model=net, + loss='categorical_crossentropy', + optimizer=keras.optimizers.SGD( + lr=0.01, + momentum=0.0, + decay=0.0, + nesterov=False), + metrics=['accuracy'], + num_gpus=num_gpus) + + #net.summary() tic = time.time() - err_top1_val, err_top5_val = validate( - acc_top1=acc_top1, - acc_top5=acc_top5, - net=net, - val_data=val_data, - batch_fn=batch_fn, - use_rec=use_rec, - dtype=dtype, - ctx=ctx) + score = net.evaluate_generator( + generator=val_gen, + steps=(val_size // batch_size), + verbose=True) if calc_weight_count: - weight_count = calc_net_weight_count(net) + weight_count = keras.utils.layer_utils.count_params(net.trainable_weights) logging.info('Model: {} trainable parameters'.format(weight_count)) - if extended_log: - logging.info('Test: err-top1={top1:.4f} ({top1})\terr-top5={top5:.4f} ({top5})'.format( - top1=err_top1_val, top5=err_top5_val)) - else: - logging.info('Test: err-top1={top1:.4f}\terr-top5={top5:.4f}'.format( - top1=err_top1_val, top5=err_top5_val)) + # if extended_log: + # logging.info('Test: err-top1={top1:.4f} ({top1})\terr-top5={top5:.4f} ({top5})'.format( + # top1=err_top1_val, top5=err_top5_val)) + # else: + # logging.info('Test: err-top1={top1:.4f}\terr-top5={top5:.4f}'.format( + # top1=err_top1_val, top5=err_top5_val)) logging.info('Time cost: {:.4f} sec'.format( time.time() - tic)) + logging.info('score: {}'.format(score)) + logging.info('Test score: {}'.format(score[0])) + logging.info('Test accuracy: {}'.format(score[1])) def main(): @@ -141,7 +194,7 @@ def main(): log_packages=args.log_packages, log_pip_packages=args.log_pip_packages) - ctx, batch_size = prepare_mx_context( + batch_size = prepare_ke_context( num_gpus=args.num_gpus, batch_size=args.batch_size) @@ -150,34 +203,35 @@ def main(): model_name=args.model, classes=num_classes, use_pretrained=args.use_pretrained, - pretrained_model_file_path=args.resume.strip(), - dtype=args.dtype, - tune_layers="", - ctx=ctx) - - if args.use_rec: - train_data, val_data, batch_fn = get_data_rec( - rec_train=args.rec_train, - rec_train_idx=args.rec_train_idx, - rec_val=args.rec_val, - rec_val_idx=args.rec_val_idx, - batch_size=batch_size, - num_workers=args.num_workers) - else: - train_data, val_data, batch_fn = get_data_loader( - data_dir=args.data_dir, - batch_size=batch_size, - num_workers=args.num_workers) + pretrained_model_file_path=args.resume.strip()) + + train_data, val_data = get_data_rec( + rec_train=args.rec_train, + rec_train_idx=args.rec_train_idx, + rec_val=args.rec_val, + rec_val_idx=args.rec_val_idx, + batch_size=batch_size, + num_workers=args.num_workers) + + # train_gen = get_data( + # it=train_data, + # batch_size=batch_size, + # num_classes=num_classes, + # report_speed=True) + val_gen = get_data( + it=val_data, + batch_size=batch_size, + num_classes=num_classes, + report_speed=True) + val_size = 50000 assert (args.use_pretrained or args.resume.strip()) test( net=net, - val_data=val_data, - batch_fn=batch_fn, - use_rec=args.use_rec, - dtype=args.dtype, - ctx=ctx, - # calc_weight_count=(not log_file_exist), + val_gen=val_gen, + val_size=val_size, + batch_size=batch_size, + num_gpus=args.num_gpus, calc_weight_count=True, extended_log=True) diff --git a/keras_/utils.py b/keras_/utils.py index 121f30c0b..7417e3c0e 100644 --- a/keras_/utils.py +++ b/keras_/utils.py @@ -1,14 +1,77 @@ import logging import os -import numpy as np -import torch.utils.data -import torchvision.transforms as transforms -import torchvision.datasets as datasets +import keras +import mxnet as mx from .model_provider import get_model +def prepare_ke_context(num_gpus, + batch_size): + batch_size *= max(1, num_gpus) + return batch_size + + +def get_data_rec(rec_train, + rec_train_idx, + rec_val, + rec_val_idx, + batch_size, + num_workers): + rec_train = os.path.expanduser(rec_train) + rec_train_idx = os.path.expanduser(rec_train_idx) + rec_val = os.path.expanduser(rec_val) + rec_val_idx = os.path.expanduser(rec_val_idx) + jitter_param = 0.4 + lighting_param = 0.1 + mean_rgb = [123.68, 116.779, 103.939] + std_rgb = [58.393, 57.12, 57.375] + + train_data = mx.io.ImageRecordIter( + path_imgrec=rec_train, + path_imgidx=rec_train_idx, + preprocess_threads=num_workers, + shuffle=True, + batch_size=batch_size, + + data_shape=(3, 224, 224), + mean_r=mean_rgb[0], + mean_g=mean_rgb[1], + mean_b=mean_rgb[2], + std_r=std_rgb[0], + std_g=std_rgb[1], + std_b=std_rgb[2], + rand_mirror=True, + random_resized_crop=True, + max_aspect_ratio=(4. / 3.), + min_aspect_ratio=(3. / 4.), + max_random_area=1, + min_random_area=0.08, + brightness=jitter_param, + saturation=jitter_param, + contrast=jitter_param, + pca_noise=lighting_param, + ) + val_data = mx.io.ImageRecordIter( + path_imgrec=rec_val, + path_imgidx=rec_val_idx, + preprocess_threads=num_workers, + shuffle=False, + batch_size=batch_size, + + resize=256, + data_shape=(3, 224, 224), + mean_r=mean_rgb[0], + mean_g=mean_rgb[1], + mean_b=mean_rgb[2], + std_r=std_rgb[0], + std_g=std_rgb[1], + std_b=std_rgb[2], + ) + return train_data, val_data + + def prepare_model(model_name, classes, use_pretrained, @@ -22,8 +85,26 @@ def prepare_model(model_name, assert (os.path.isfile(pretrained_model_file_path)) logging.info('Loading model: {}'.format(pretrained_model_file_path)) net.load_weights(filepath=pretrained_model_file_path) - #from keras.models import load_model - #net = load_model(pretrained_model_file_path) - return net + + +def backend_agnostic_compile(model, + loss, + optimizer, + metrics, + num_gpus): + if keras.backend._backend == 'mxnet': + gpu_list = ["gpu(%d)" % i for i in range(num_gpus)] + model.compile( + loss=loss, + optimizer=optimizer, + metrics=metrics, + context=gpu_list) + else: + if num_gpus > 1: + print("Warning: num_gpus > 1 but not using MxNet backend") + model.compile( + loss=loss, + optimizer=optimizer, + metrics=metrics)