Merge branch 'release-1.3'

bugfixes and feature updates

.gitignore

@@ -132,7 +132,8 @@ dmypy.json
 .idea/
 
 # TensorFlow models
-save_model/
+saves/model/*.h5
+saves/block/*.json
 
 # Others
 others/

README.md

@@ -21,11 +21,11 @@ To build and run FedCoin, you must ensure the following software or package(s) a
 * [Python 3.7](https://www.python.org/downloads/)
 * [TensorFlow 2.3.0](https://www.tensorflow.org/install) (Mirror: [Google China](https://tensorflow.google.cn/install))
 
-Moreover, Linux distribution like [Ubuntu](https://ubuntu.com/download) and [Kubuntu](https://kubuntu.org/getkubuntu/) are recommended.
+Moreover, Linux distribution like [Ubuntu 16.04.7 LTS](https://releases.ubuntu.com/16.04.7/) is recommended as the operating system. [Anaconda](https://www.anaconda.com/products/individual#Downloads) (Mirror: [TUNA](https://mirrors.tuna.tsinghua.edu.cn/anaconda/archive/)) is recommended as the environment manager. Run the `conda create -n fedcoin python=3.5 tensorflow-gpu` command, and the `python -m pip install docker` command in new created `fedcoin` environment could easily get a basic running environment.
 
 ### Generate weights
 
-Firstly, you should create a directory named `save_model`, then run the following codes in the Python environment to generate weights:
+Run the following codes in the Python environment to generate weights:
 ```python
 from blockchain.application.fedcoin import FedCoin
 FedCoin.run_fl_server()
@@ -52,8 +52,11 @@ Note: The condition "TRAIN_PRICE + COM_PRICE + SAP_PRICE = 1" should be satisfie
 
 The shell code `python main.py` will build `fedcoin` and `fedcoin_lw` images, and run several `fedcoin` containers that matches the number of replicas defined in settings.
 
+Run `docker run fedcoin_lw` will start publishing tasks.
+
 The batch file `clean.sh` will stop and delete containers, as well as delete images that are untagged.
 
 ## References
 
-* Yuan Liu, Shuai Sun, Zhengpeng Ai, Shuangfeng Zhang, Zelei Liu, Han Yu. FedCoin: A Peer-to-Peer Payment System for Federated Learning ([arXiv:2002.11711](https://arxiv.org/abs/2002.11711))
+* Yuan Liu, Zhengpeng Ai, Shuai Sun, Shuangfeng Zhang, Zelei Liu, Han Yu. FedCoin: A Peer-to-Peer Payment System for Federated Learning. In Book "Federated Learning: Privacy and Incentive" ([DOI:10.1007/978-3-030-63076-8_9](https://doi.org/10.1007/978-3-030-63076-8_9))
+* Yuan Liu, Shuai Sun, Zhengpeng Ai, Shuangfeng Zhang, Zelei Liu, Han Yu. FedCoin: A Peer-to-Peer Payment System for Federated Learning. In arXiv ([arXiv:2002.11711](https://arxiv.org/abs/2002.11711))

blockchain/application/fedcoin.py

@@ -2,10 +2,10 @@
 from blockchain.consensus.posap import PoSapBlock, PoSapMessageHandler, TaskMessage
 from blockchain.application.application import Application
 from blockchain.util.settings import *
-from blockchain.util.printer import Printer
+from blockchain.util.dataset_loader import *
 import base64
 import math
-import tensorflow as tf
+import keras
 
 
 class FedCoin(Application):
@@ -57,21 +57,21 @@ def run_lightweight_node(self):
 
     @staticmethod
     def run_fl_server():
-        (train_images, train_labels), (test_images, test_labels) = tf.keras.datasets.mnist.load_data()
+        (train_images, train_labels), (test_images, test_labels) = DatasetLoader.load_mnist()
 
         # Normalize pixel values to be between 0 and 1
         train_images, test_images = train_images / 255.0, test_images / 255.0
 
-        model = tf.keras.models.Sequential([
-            tf.keras.layers.Flatten(input_shape=(28, 28)),
-            tf.keras.layers.Dense(128, activation='relu'),
-            tf.keras.layers.Dense(10)
+        model = keras.models.Sequential([
+            keras.layers.Flatten(input_shape=(28, 28)),
+            keras.layers.Dense(128, activation='relu'),
+            keras.layers.Dense(10)
         ])
 
         # model.summary()
 
-        model.compile(optimizer=tf.keras.optimizers.SGD(learning_rate=0.1),
-                      loss=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True),
+        model.compile(optimizer='sgd',
+                      loss='sparse_categorical_crossentropy',
                       metrics=['accuracy'])
 
         init_weight = model.get_weights()
@@ -81,7 +81,7 @@ def run_fl_server():
             model.fit(train_images[i * client_size:(i + 1) * client_size],
                       train_labels[i * client_size:(i + 1) * client_size], epochs=10)
 
-            model.save('save_model/model_' + str(i + 1) + '.h5')
+            model.save('saves/model/' + str(i + 1) + '.h5')
 
             model.set_weights(init_weight)
 
@@ -90,5 +90,5 @@ def run_fl_server():
             # print('\nTest accuracy:', test_acc)
 
         del init_weight
-        tf.keras.backend.clear_session()
+        keras.backend.clear_session()
         return

blockchain/consensus/posap.py

@@ -3,14 +3,15 @@
 from blockchain.consensus.consensus import Consensus
 from blockchain.util.settings import *
 from blockchain.util.downloader import Downloader
+from blockchain.util.dataset_loader import DatasetLoader
 import base64
 import copy
 import hashlib
 import math
 import numpy as np
 import random
 import struct
-import tensorflow as tf
+import keras
 
 
 # Data Layer
@@ -135,7 +136,7 @@ def run(self):
             model_urls = self.msg_dict['model_urls']
             i = 0
             for url in model_urls:
-                d = Downloader(url, 'save_model/model_' + str(i + 1) + '.h5')
+                d = Downloader(url, 'saves/model/' + str(i + 1) + '.h5')
                 d.start()
                 d.join()
                 i += 1
@@ -149,12 +150,12 @@ def run(self):
             self.app.set_var('received', False)
             self.msg_handler.lock.release()
             for i in range(K):
-                model = tf.keras.models.load_model('save_model/model_' + str(i + 1) + '.h5')
+                model = keras.models.load_model('saves/model/' + str(i + 1) + '.h5')
                 self.msg_handler.lock.acquire()
                 weights_list.append(copy.deepcopy(model.get_weights()))
                 self.msg_handler.lock.release()
                 del model
-                tf.keras.backend.clear_session()
+                keras.backend.clear_session()
             PoSap(self.msg_handler).run(self.msg_dict['timestamp'], runtime)
             msg = ShapleyMessage(self.app.get_var('s'))
             self.network.send(msg)
@@ -223,21 +224,21 @@ def run(self):
 
 # Consensus Layer
 class PoSap(Consensus):
-    (train_images, train_labels), (test_images, test_labels) = tf.keras.datasets.mnist.load_data()
+    (train_images, train_labels), (test_images, test_labels) = DatasetLoader.load_mnist()
 
     # Normalize pixel values to be between 0 and 1
     train_images, test_images = train_images / 255.0, test_images / 255.0
 
-    model = tf.keras.models.Sequential([
-        tf.keras.layers.Flatten(input_shape=(28, 28)),
-        tf.keras.layers.Dense(128, activation='relu'),
-        tf.keras.layers.Dense(10)
+    model = keras.models.Sequential([
+        keras.layers.Flatten(input_shape=(28, 28)),
+        keras.layers.Dense(128, activation='relu'),
+        keras.layers.Dense(10)
     ])
 
     # model.summary()
 
-    model.compile(optimizer=tf.keras.optimizers.SGD(learning_rate=0.1),
-                  loss=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True),
+    model.compile(optimizer='sgd',
+                  loss='sparse_categorical_crossentropy',
                   metrics=['accuracy'])
 
     def __init__(self, msg_handler: MessageHandler):
@@ -260,11 +261,11 @@ def run(self, timestamp: float, runtime: float = RUNTIME):
             self.lock.release()
             weights = PoSap.aggregate(self.weights_list[r[0]])
             s_t[r[0]] = PoSap.calc_accuracy(weights)
-            tf.keras.backend.clear_session()
+            keras.backend.clear_session()
             for i in range(1, K):
                 weights = PoSap.aggregate(PoSap.aggregate(self.weights_list[r[i]]), weights, i)
                 s_t[r[i]] = PoSap.calc_accuracy(weights)
-                tf.keras.backend.clear_session()
+                keras.backend.clear_session()
                 tmp = 0
                 for j in range(0, i):
                     tmp += s_t[r[j]]

blockchain/util/dataset_loader.py

@@ -0,0 +1,8 @@
+import numpy as np
+
+
+class DatasetLoader:
+    @staticmethod
+    def load_mnist(path='saves/dataset/mnist.npz'):
+        data = np.load(path)
+        return (data['x_train'], data['y_train']), (data['x_test'], data['y_test'])

saves/block/README.txt

@@ -0,0 +1 @@
+Reserved for local blocks.

saves/dataset/README.txt

@@ -0,0 +1 @@
+Reserved for common datasets.

saves/model/README.txt

@@ -0,0 +1 @@
+Reserved for training models.