Merge pull request #1431 from anikitinDSR/rc-1.9.2.rc1

[rc-1.9.2.rc1] Release candidate for indy-node

build-scripts/ubuntu-1604/prepare-package.sh

@@ -22,7 +22,6 @@ echo -e "\n\nPrepares indy-plenum debian package version"
 sed -i -r "s~indy-plenum==([0-9\.]+[0-9])(\.)?([a-z]+)~indy-plenum==\1\~\3~" setup.py
 
 echo -e "\nAdapt the dependencies for the Canonical archive"
-sed -i "s~python-dateutil~python3-dateutil~" setup.py
 sed -i "s~timeout-decorator~python3-timeout-decorator~" setup.py
 sed -i "s~distro~python3-distro~" setup.py
 

ci/ubuntu.dockerfile

@@ -8,7 +8,7 @@ ARG venv=venv
 RUN apt-get update -y && apt-get install -y \
     python3-nacl \
     libindy-crypto=0.4.5 \
-    libindy=1.10.0~1198 \
+    libindy=1.11.0~1282 \
 # rocksdb python wrapper
     libbz2-dev \
     zlib1g-dev \

data/migrations/deb/1_9_1_to_1_9_2.py

@@ -0,0 +1,82 @@
+import os
+import shutil
+import subprocess
+import sys
+
+from indy_common.config_helper import NodeConfigHelper
+from indy_common.config_util import getConfig
+from stp_core.common.log import getlogger
+
+
+logger = getlogger()
+
+ENV_FILE_PATH = "/etc/indy/indy.env"
+BUILDER_NET_NETWORK_NAME = "net3"
+
+
+def get_node_name():
+    node_name = None
+    node_name_key = 'NODE_NAME'
+
+    if os.path.exists(ENV_FILE_PATH):
+        with open(ENV_FILE_PATH, "r") as fenv:
+            for line in fenv.readlines():
+                if line.find(node_name_key) != -1:
+                    node_name = line.split('=')[1].strip()
+                    break
+    if node_name is None:
+        logger.error("{} file doesn't contains a node name. Please add string like NODE_NAME=<node name> into this file".format(ENV_FILE_PATH))
+
+    return node_name
+
+
+def remove_dir(path_to_dir):
+    try:
+        shutil.rmtree(path_to_dir)
+    except shutil.Error as ex:
+        logger.error("""While removing directory: {} the next error was raised:
+{}""".format(path_to_dir, ex))
+
+        return False
+    return True
+
+
+def migrate_all():
+
+    node_name = get_node_name()
+    if node_name is None:
+        return False
+
+    config = getConfig()
+    config_helper = NodeConfigHelper(node_name, config)
+
+    if BUILDER_NET_NETWORK_NAME != config.NETWORK_NAME:
+        logger.info("This script can be used only for {} network".format(BUILDER_NET_NETWORK_NAME))
+        return False
+
+    path_to_config_state = os.path.join(config_helper.ledger_dir, config.configStateDbName)
+    path_to_config_ts_db = os.path.join(config_helper.ledger_dir, config.configStateTsDbName)
+
+    if not os.path.exists(path_to_config_ts_db):
+        logger.error("Path {} to config's timestamp storage does not exist".format(path_to_config_ts_db))
+        return False
+
+    if not os.path.exists(path_to_config_state):
+        logger.error("Path {} to config_state storage does not exist".format(path_to_config_state))
+        return False
+
+    if not remove_dir(path_to_config_ts_db):
+        logger.error("Failed to remove {}".format(path_to_config_ts_db))
+        return False
+
+    if not remove_dir(path_to_config_state):
+        logger.error("Failed to remove {}".format(path_to_config_state))
+        return False
+
+    logger.info("Config state storage was successfully removed. Path was {}".format(path_to_config_state))
+
+    return True
+
+
+if not migrate_all():
+    logger.error("Config state storage removing failed")

design/transaction_endorser.md

@@ -8,6 +8,7 @@ As a transaction author, I need my transactions to be written to the ledger pres
  - A transaction author can use a different transaction endorser for future transactions, including updates to attribs and key rotations.
  - The transaction must use the author key to sign the transaction author agreement 
  - If the endorser field is included in a transaction, then the ledger will reject the transaction if it is not signed by the endorser.
+ - It should not be possible to endorse a transaction without explicitly specifying the Endorser.
 
 ## Proposed workflow
 1. Transaction Author builds a new request (`indy_build_xxx_reqeust`).
@@ -62,6 +63,12 @@ pub extern fn indy_append_request_endorser(command_handle: CommandHandle,
   - there must be `identifier`'s signature in `signatures`
   - `signature` must be absent
 
+#### Request dynamic validation
+In order to avoid endorsement without explicitly specifying an Endorser, the following changes to dynamic validation must be implemented: 
+- If there is `endorser` field, then it can have Endorser role only.
+- If request is multi-signed, and the author is not Trustee/Steward/Endorser, then `endorser` field must be present
+    - `endorser` field is not required if the author is already an Endorser or a trusted role, since multiple trusted signatures (3 Trustees for example) may be required to send a transaction. 
+
 #### Signature Verification
 No changes are required. 
 

indy_common/authorize/auth_constraints.py

@@ -84,7 +84,7 @@ def from_dict(as_dict):
 class AuthConstraint(AbstractAuthConstraint):
     def __init__(self, role, sig_count,
                  need_to_be_owner=False,
-                 off_ledger_signature=False,
+                 off_ledger_signature=None,
                  metadata={}):
         self._role_validation(role, off_ledger_signature)
         self.role = role
@@ -96,14 +96,16 @@ def __init__(self, role, sig_count,
 
     @property
     def as_dict(self):
-        return {
+        constraint = {
             CONSTRAINT_ID: self.constraint_id,
             ROLE: self.role,
             SIG_COUNT: self.sig_count,
             NEED_TO_BE_OWNER: self.need_to_be_owner,
-            OFF_LEDGER_SIGNATURE: self.off_ledger_signature,
             METADATA: self.metadata
         }
+        if self.off_ledger_signature is not None:
+            constraint[OFF_LEDGER_SIGNATURE] = self.off_ledger_signature
+        return constraint
 
     @staticmethod
     def _role_validation(role, off_ledger_signature):
@@ -155,7 +157,7 @@ def _metadata_str(self):
     def from_dict(as_dict):
         return AuthConstraint(role=as_dict[ROLE], sig_count=as_dict[SIG_COUNT],
                               need_to_be_owner=as_dict.get(NEED_TO_BE_OWNER, False),
-                              off_ledger_signature=as_dict.get(OFF_LEDGER_SIGNATURE, False),
+                              off_ledger_signature=as_dict.get(OFF_LEDGER_SIGNATURE, None),
                               metadata=as_dict.get(METADATA, {}))
 
     def set_metadata(self, metadata: dict):

indy_common/authorize/auth_request_validator.py

@@ -4,7 +4,7 @@
 from indy_common.authorize.auth_actions import AbstractAuthAction
 from indy_common.authorize.auth_constraints import ConstraintsEnum, AbstractConstraintSerializer
 from indy_common.authorize.authorizer import AbstractAuthorizer, CompositeAuthorizer, RolesAuthorizer, AndAuthorizer, \
-    OrAuthorizer, AuthValidationError, ForbiddenAuthorizer
+    OrAuthorizer, AuthValidationError, ForbiddenAuthorizer, EndorserAuthorizer
 from indy_common.constants import LOCAL_AUTH_POLICY, CONFIG_LEDGER_AUTH_POLICY
 from indy_common.types import Request
 from indy_node.persistence.idr_cache import IdrCache
@@ -13,7 +13,6 @@
 from state.pruning_state import PruningState
 from stp_core.common.log import getlogger
 
-
 logger = getlogger()
 
 
@@ -44,7 +43,10 @@ def __init__(self,
         self.register_default_authorizers()
 
     def register_default_authorizers(self):
-        self.register_authorizer(RolesAuthorizer(cache=self.cache), auth_constraint_id=ConstraintsEnum.ROLE_CONSTRAINT_ID)
+        self.register_authorizer(RolesAuthorizer(cache=self.cache),
+                                 auth_constraint_id=ConstraintsEnum.ROLE_CONSTRAINT_ID)
+        self.register_authorizer(EndorserAuthorizer(cache=self.cache),
+                                 auth_constraint_id=ConstraintsEnum.ROLE_CONSTRAINT_ID)
         self.register_authorizer(AndAuthorizer(), auth_constraint_id=ConstraintsEnum.AND_CONSTRAINT_ID)
         self.register_authorizer(OrAuthorizer(), auth_constraint_id=ConstraintsEnum.OR_CONSTRAINT_ID)
         self.register_authorizer(ForbiddenAuthorizer(), auth_constraint_id=ConstraintsEnum.FORBIDDEN_CONSTRAINT_ID)

indy_common/authorize/authorizer.py

@@ -7,11 +7,12 @@
 from indy_common.authorize.auth_constraints import AbstractAuthConstraint, AuthConstraint, \
     AuthConstraintAnd, ConstraintsEnum, AuthConstraintOr, AuthConstraintForbidden
 from indy_common.authorize.helper import get_named_role
-from indy_common.constants import NYM, CLAIM_DEF
+from indy_common.constants import ENDORSER, ENDORSER_STRING
 from indy_common.roles import Roles
 from indy_common.transactions import IndyTransactions
 from indy_common.types import Request
 from indy_node.persistence.idr_cache import IdrCache
+from plenum.common.constants import STEWARD, TRUSTEE, TRUSTEE_STRING, STEWARD_STRING
 
 logger = getLogger()
 
@@ -37,6 +38,7 @@ def authorize(self,
 
 
 class RolesAuthorizer(AbstractAuthorizer):
+
     def __init__(self, cache: IdrCache):
         super().__init__()
         self.cache = cache
@@ -97,12 +99,8 @@ def authorize(self,
                   request: Request,
                   auth_constraint: AuthConstraint,
                   auth_action: AbstractAuthAction = None):
-        if auth_constraint.sig_count > 0 and not auth_constraint.off_ledger_signature and self.get_role(
-                request) is None:
-            return False, "sender's DID {} is not found in the Ledger".format(request.identifier)
-        if not self.is_sig_count_accepted(request, auth_constraint):
-            role = Roles(auth_constraint.role).name if auth_constraint.role != '*' else '*'
-            return False, "Not enough {} signatures".format(role)
+        # 1. Check that the Author is the owner
+        # do first since it doesn't require going to state
         if not self.is_owner_accepted(auth_constraint, auth_action):
             if auth_action.field != '*':
                 return False, "{} can not touch {} field since only the owner can modify it". \
@@ -112,6 +110,22 @@ def authorize(self,
                 return False, "{} can not edit {} txn since only owner can modify it". \
                     format(self.get_named_role_from_req(request),
                            IndyTransactions.get_name_from_code(auth_action.txn_type))
+
+        author_role = self.get_role(request)
+
+        # 2. Check that the Author is present on the ledger
+        if auth_constraint.sig_count > 0 and not auth_constraint.off_ledger_signature and author_role is None:
+            return False, "sender's DID {} is not found in the Ledger".format(request.identifier)
+
+        # 3. Check that the Author signed the transaction in case of multi-sig
+        if auth_constraint.sig_count > 0 and request.signatures and request.identifier not in request.signatures:
+            return False, "Author must sign the transaction"
+
+        # 4. Check that there are enough signatures of the needed role
+        if not self.is_sig_count_accepted(request, auth_constraint):
+            role = Roles(auth_constraint.role).name if auth_constraint.role != '*' else '*'
+            return False, "Not enough {} signatures".format(role)
+
         return True, ""
 
     def _get_role(self, idr):
@@ -192,3 +206,90 @@ def authorize(self,
                   auth_constraint: AuthConstraintForbidden,
                   auth_action: AbstractAuthAction):
         return False, str(auth_constraint)
+
+
+class EndorserAuthorizer(AbstractAuthorizer):
+    NO_NEED_FOR_ENDORSER_ROLES = {ENDORSER, TRUSTEE, STEWARD}
+    NO_NEED_FOR_ENDORSER_ROLES_STR = [ENDORSER_STRING, TRUSTEE_STRING, STEWARD_STRING]
+
+    ENDORSER_ROLES = {ENDORSER}
+    ENDORSER_ROLES_STR = {ENDORSER_STRING}
+
+    def __init__(self, cache: IdrCache):
+        super().__init__()
+        self.cache = cache
+
+    def authorize(self,
+                  request: Request,
+                  auth_constraint: AuthConstraint,
+                  auth_action: AbstractAuthAction = None):
+        # check if endorser role is valid first
+        res, reason = self._check_endorser_role(request)
+        if not res:
+            return res, reason
+
+        # check if Endorser field must be explicitly present
+        res, reason = self._check_endorser_field_presence(request)
+        if not res:
+            return res, reason
+
+        return True, ""
+
+    def _check_endorser_role(self, request):
+        if request.endorser is None:
+            return True, ""
+
+        endorser_role = self._get_role(request.endorser)
+        if endorser_role not in self.ENDORSER_ROLES:
+            return False, "Endorser must have one of the following roles: {}".format(str(self.ENDORSER_ROLES_STR))
+
+        return True, ""
+
+    def _check_endorser_field_presence(self, request):
+        # 1. Check that if Endorser is present, Endorser must sign the transaction
+        # if author=endorser, then no multi-sig is required, since he's already signed the txn
+        if request.endorser != request.identifier:
+            if request.endorser and not request.signatures:
+                return False, "Endorser must sign the transaction"
+            if request.endorser and request.endorser not in request.signatures:
+                return False, "Endorser must sign the transaction"
+
+        # 2. Endorser is required only when the transaction is endorsed, that is signed by someone else besides the author.
+        # If the auth rule requires an endorser to submit the transaction, then it will be caught by the Roles Authorizer,
+        # so no need to check anything here if we don't have any extra signatures.
+        if not request.signatures:
+            return True, ""
+        has_extra_sigs = len(request.signatures) > 1 or (request.identifier not in request.signatures)
+        if not has_extra_sigs:
+            return True, ""
+
+        # 3. Endorser is required for unprivileged roles only
+        author_role = self._get_role(request.identifier)
+        if author_role == "":  # "" - IDENTITY_OWNER
+            author_role = None
+        if author_role in self.NO_NEED_FOR_ENDORSER_ROLES:
+            return True, ""
+
+        # 4. Endorser field is not required when multi-signed by non-privileged roles only
+        has_privileged_sig = False
+        for idr, _ in request.signatures.items():
+            role = self._get_role(idr)
+            if role in self.NO_NEED_FOR_ENDORSER_ROLES:
+                has_privileged_sig = True
+                break
+        if not has_privileged_sig:
+            return True, ""
+
+        # 5. Check that Endorser field is present
+        if request.endorser is None:
+            return False, "'Endorser' field must be explicitly set for the endorsed transaction"
+
+        return True, ""
+
+    def _get_role(self, idr):
+        if idr is None:
+            return None
+        try:
+            return self.cache.getRole(idr, isCommitted=False)
+        except KeyError:
+            return None

indy_common/test/auth/metadata/conftest.py

@@ -29,6 +29,16 @@ def is_owner(request):
     return request.param
 
 
+@pytest.fixture(scope='module', params=[None, ENDORSER])
+def endorser(request):
+    return request.param
+
+
+@pytest.fixture(scope='module', params=[IDENTITY_OWNER, TRUSTEE])
+def author(request):
+    return request.param
+
+
 @pytest.fixture(
     scope='module',
     params=[role for r in range(len(ROLES) + 1) for role in combinations(ROLES, r)],