From cc82178fa1e989bd3dd9c4a9b84aaaa67ffcffd0 Mon Sep 17 00:00:00 2001 From: Michal Vasko Date: Wed, 1 Aug 2018 15:40:44 +0200 Subject: [PATCH 01/49] server CHANGE libsysrepo api changed --- server/main.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/main.c b/server/main.c index c4d02c3bd..4683d8f62 100644 --- a/server/main.c +++ b/server/main.c @@ -544,7 +544,7 @@ np2srv_feature_change_clb(const char *module_name, const char *feature_name, boo } static int -np2srv_state_data_clb(const char *xpath, sr_val_t **values, size_t *values_cnt, void *UNUSED(private_ctx)) +np2srv_state_data_clb(const char *xpath, sr_val_t **values, size_t *values_cnt, uint64_t UNUSED(request_id), void *UNUSED(private_ctx)) { struct lyd_node *data = NULL, *node, *iter; struct ly_set *set = NULL; From 476983f4de75090b15255e5be7e902716b106f2b Mon Sep 17 00:00:00 2001 From: Michal Vasko Date: Thu, 2 Aug 2018 09:45:56 +0200 Subject: [PATCH 02/49] server BUGFIX invalid cleanup order Fixes #306 --- server/main.c | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/server/main.c b/server/main.c index 4683d8f62..f294c458f 100644 --- a/server/main.c +++ b/server/main.c @@ -1566,10 +1566,6 @@ main(int argc, char *argv[]) if (np2srv.sr_subscr) { sr_unsubscribe(np2srv.sr_sess.srs, np2srv.sr_subscr); } - if (np2srv.sr_sess.srs) { - sr_session_stop(np2srv.sr_sess.srs); - } - sr_disconnect(np2srv.sr_conn); /* libnetconf2 cleanup */ if (np2srv.nc_ps) { @@ -1577,6 +1573,9 @@ main(int argc, char *argv[]) } nc_ps_free(np2srv.nc_ps); + /* clears all the sessions also */ + sr_disconnect(np2srv.sr_conn); + nc_server_destroy(); /* monitoring cleanup */ From 93e6d33b84310568000825c15147a32c66b5fe72 Mon Sep 17 00:00:00 2001 From: Michal Vasko Date: Thu, 2 Aug 2018 09:48:24 +0200 Subject: [PATCH 03/49] server VERSION bump to version 0.6.0 --- server/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/CMakeLists.txt b/server/CMakeLists.txt index b79f19d9c..11e8ce633 100644 --- a/server/CMakeLists.txt +++ b/server/CMakeLists.txt @@ -13,7 +13,7 @@ if(NOT UNIX) endif() # set version -set(NP2SRV_VERSION 0.5.28) +set(NP2SRV_VERSION 0.6.0) # set default build type if not specified by user if(NOT CMAKE_BUILD_TYPE) From bbf4b07fe536749071ecd898b09f0728aded2729 Mon Sep 17 00:00:00 2001 From: Michal Vasko Date: Mon, 6 Aug 2018 08:30:23 +0200 Subject: [PATCH 04/49] server BUGFIX handle session termination correctly Fixes #308 --- server/main.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/main.c b/server/main.c index f294c458f..ddef83e06 100644 --- a/server/main.c +++ b/server/main.c @@ -1287,7 +1287,7 @@ worker_thread(void *arg) /* listen for incoming requests on active NETCONF sessions */ rc = nc_ps_poll(np2srv.nc_ps, 0, &ncs); - if (rc & (NC_PSPOLL_NOSESSIONS | NC_PSPOLL_TIMEOUT | NC_PSPOLL_ERROR)) { + if ((rc & (NC_PSPOLL_NOSESSIONS | NC_PSPOLL_TIMEOUT | NC_PSPOLL_ERROR)) && !(rc & NC_PSPOLL_SESSION_TERM)) { /* if there is no active session, timeout, or an error, rest for a while */ pthread_rwlock_unlock(&np2srv.ly_ctx_lock); np_sleep(10); From 93bfdb4cf2db997a6e55283cb948f60287173e48 Mon Sep 17 00:00:00 2001 From: Michal Vasko Date: Mon, 6 Aug 2018 08:31:16 +0200 Subject: [PATCH 05/49] server VERSION bum to version 0.6.1 --- server/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/CMakeLists.txt b/server/CMakeLists.txt index 11e8ce633..33e3daf4f 100644 --- a/server/CMakeLists.txt +++ b/server/CMakeLists.txt @@ -13,7 +13,7 @@ if(NOT UNIX) endif() # set version -set(NP2SRV_VERSION 0.6.0) +set(NP2SRV_VERSION 0.6.1) # set default build type if not specified by user if(NOT CMAKE_BUILD_TYPE) From 4c85abd49fc2d8b705d0278540569ac4943efddb Mon Sep 17 00:00:00 2001 From: Andrew Langefeld Date: Mon, 6 Aug 2018 20:03:16 -0500 Subject: [PATCH 06/49] Add unit test that deletes integer leaf This commit adds a unit test to test deleting a leaf whose type is an integer. (This functionality was broken beginning in libyang@15e3e61. The test will not pass until the bug is resolved.) --- server/tests/test_edit_get_config.c | 74 +++++++++++++++++++++++++++++ 1 file changed, 74 insertions(+) mode change 100644 => 100755 server/tests/test_edit_get_config.c diff --git a/server/tests/test_edit_get_config.c b/server/tests/test_edit_get_config.c old mode 100644 new mode 100755 index f9f0ae152..68a67c72d --- a/server/tests/test_edit_get_config.c +++ b/server/tests/test_edit_get_config.c @@ -596,6 +596,7 @@ np_start(void **state) "false" "disabled" "" + "2000" "
" "10.0.0.5" "255.0.0.0" @@ -710,6 +711,7 @@ test_edit_delete1(void **state) "false" "disabled" "" + "2000" "
" "10.0.0.5" "255.0.0.0" @@ -782,6 +784,7 @@ test_edit_delete2(void **state) "false" "disabled" "" + "2000" "
" "10.0.0.5" "255.0.0.0" @@ -860,6 +863,76 @@ test_edit_delete3(void **state) test_read(p_in, edit_rpl, __LINE__); } +static void +test_edit_delete4(void **state) +{ + (void)state; /* unused */ + const char *get_config_rpc = + "" + "" + "" + "" + "" + "" + ""; + const char *get_config_rpl = + "" + "" +"" + "" + "iface2" + "iface2 dsc" + "ianaift:softwareLoopback" + "false" + "disabled" + "" + "
" + "10.0.0.5" + "255.0.0.0" + "
" + "
" + "172.0.0.5" + "16" + "
" + "" + "10.0.0.1" + "01:34:56:78:9a:bc:de:fa" + "" + "" + "" +"" + "" + ""; + const char *edit_rpc = + "" + "" + "" + "" + "" + "" + "" + "" + "iface2" + "" + "" + "" + "" + "" + "" + "" + ""; + const char *edit_rpl = + "" + "" + ""; + + test_write(p_out, edit_rpc, __LINE__); + test_read(p_in, edit_rpl, __LINE__); + + test_write(p_out, get_config_rpc, __LINE__); + test_read(p_in, get_config_rpl, __LINE__); +} + static void test_edit_create1(void **state) { @@ -1613,6 +1686,7 @@ main(void) cmocka_unit_test(test_edit_delete1), cmocka_unit_test(test_edit_delete2), cmocka_unit_test(test_edit_delete3), + cmocka_unit_test(test_edit_delete4), cmocka_unit_test(test_edit_create1), cmocka_unit_test(test_edit_create2), cmocka_unit_test(test_edit_create3), From 7c7a38513a4b913e43a90ae61e2ab986d1b01b3c Mon Sep 17 00:00:00 2001 From: Michal Vasko Date: Tue, 7 Aug 2018 11:51:08 +0200 Subject: [PATCH 07/49] keystored CHANGE update callback api --- keystored/keystored.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/keystored/keystored.c b/keystored/keystored.c index 466d5b760..822174c81 100644 --- a/keystored/keystored.c +++ b/keystored/keystored.c @@ -121,7 +121,7 @@ ks_cert_change_cb(sr_session_ctx_t *UNUSED(session), const char *UNUSED(module_n } static int -ks_privkey_get_cb(const char *xpath, sr_val_t **values, size_t *values_cnt, void *UNUSED(private_ctx)) +ks_privkey_get_cb(const char *xpath, sr_val_t **values, size_t *values_cnt, uint64_t UNUSED(request_id), void *UNUSED(private_ctx)) { int ret; const char *name; From 8c178d2d75121b9e48a054f8d6f623c574587d00 Mon Sep 17 00:00:00 2001 From: Michal Vasko Date: Tue, 7 Aug 2018 13:33:30 +0200 Subject: [PATCH 08/49] server DOC add incorrect filter to known issues --- server/KNOWNISSUES.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/server/KNOWNISSUES.md b/server/KNOWNISSUES.md index 33aacdc40..af1639cdc 100644 --- a/server/KNOWNISSUES.md +++ b/server/KNOWNISSUES.md @@ -6,3 +6,11 @@ pthread_rwlockattr_setkind_np() and the number of worker threads is increased (via cmake THREAD_COUNT variable), the thread processing the modules changes in sysrepo (module install/uninstall or feature changes) can starve by waiting for lock to wite changes into the netopeer's context. + +XPath filter limitations +------------------------ + +Correct filter result is guaranteed only when all the filtered nodes +are only from one YANG schema and no unions are used. Otherwise, +the or may finish with an error or possibly +less data than would be correct. From 3f2976a084069bb42eccd3d51cd968aed73700c8 Mon Sep 17 00:00:00 2001 From: Andrew Langefeld Date: Wed, 8 Aug 2018 09:01:50 -0500 Subject: [PATCH 09/49] Fix issue with actions from augments Before this change an action added by an augment would always be treated as "unimplemented", even if there is a subscriber for it downstream of sysrepo. This patch causes the failing test cases (see [1] and [2]) to start passing. This was not added to a unit test as main.c isn't being tested in that context right now. [1]: https://travis-ci.org/ADTRAN/netopeer2-integration-tests/builds/413288159#L7278 [2]: https://github.com/ADTRAN/netopeer2-integration-tests/blob/master/tests/test_action.py#L58 --- server/main.c | 29 +++++++++++++++++++++-------- 1 file changed, 21 insertions(+), 8 deletions(-) diff --git a/server/main.c b/server/main.c index ddef83e06..5b70d9963 100644 --- a/server/main.c +++ b/server/main.c @@ -222,18 +222,13 @@ signal_handler(int sig) } } -static int -np2srv_module_assign_clbs(const struct lys_module *mod) +static void +np2srv_node_assign_clbs(struct lys_node *start) { struct lys_node *snode, *next; - if (!strcmp(mod->name, "ietf-netconf-monitoring") || !strcmp(mod->name, "ietf-netconf")) { - /* skip it, use internal implementations from libnetconf2 */ - return EXIT_SUCCESS; - } - /* set RPC and Notifications callbacks */ - LY_TREE_DFS_BEGIN(mod->data, next, snode) { + LY_TREE_DFS_BEGIN(start, next, snode) { if (snode->nodetype & (LYS_RPC | LYS_ACTION)) { nc_set_rpc_callback(snode, op_generic); goto dfs_nextsibling; @@ -262,6 +257,24 @@ np2srv_module_assign_clbs(const struct lys_module *mod) } } +} + +static int +np2srv_module_assign_clbs(const struct lys_module *mod) +{ + if (!strcmp(mod->name, "ietf-netconf-monitoring") || !strcmp(mod->name, "ietf-netconf")) { + /* skip it, use internal implementations from libnetconf2 */ + return EXIT_SUCCESS; + } + np2srv_node_assign_clbs(mod->data); + for(uint8_t i = 0; i < mod->augment_size; ++i) { + struct lys_node *target = mod->augment[i].target; + if (!strcmp(target->module->name, "ietf-netconf-monitoring") || !strcmp(target->module->name, "ietf-netconf")) { + continue; + } + np2srv_node_assign_clbs(target); + } + return EXIT_SUCCESS; } From bd59c033c9f753e29e191038eabcbdc6a8f17c53 Mon Sep 17 00:00:00 2001 From: Andrew Langefeld Date: Wed, 8 Aug 2018 14:15:54 -0500 Subject: [PATCH 10/49] Add -std=gnu11 to C flags This matches sysrepo and libyang, which already use -std=gnu11. (The previous commit added a construct only available starting in C99, specifically, a for loop initial declaration. This construct is widely used in sysrepo already.) --- server/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) mode change 100644 => 100755 server/CMakeLists.txt diff --git a/server/CMakeLists.txt b/server/CMakeLists.txt old mode 100644 new mode 100755 index 33e3daf4f..26ce32361 --- a/server/CMakeLists.txt +++ b/server/CMakeLists.txt @@ -20,7 +20,7 @@ if(NOT CMAKE_BUILD_TYPE) set(CMAKE_BUILD_TYPE debug) endif() -set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wall -Wextra") +set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wall -Wextra -std=gnu11") set(CMAKE_C_FLAGS_RELEASE "-O2 -DNDEBUG") set(CMAKE_C_FLAGS_DEBUG "-g -O0 -DDEBUG") From a38ebb80ea57be69ffe715a33a4567a088a249e2 Mon Sep 17 00:00:00 2001 From: Andrew Langefeld Date: Wed, 8 Aug 2018 14:20:57 -0500 Subject: [PATCH 11/49] Minor style change to match contributing guide --- server/main.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/main.c b/server/main.c index 5b70d9963..ca95be317 100644 --- a/server/main.c +++ b/server/main.c @@ -267,7 +267,7 @@ np2srv_module_assign_clbs(const struct lys_module *mod) return EXIT_SUCCESS; } np2srv_node_assign_clbs(mod->data); - for(uint8_t i = 0; i < mod->augment_size; ++i) { + for (uint8_t i = 0; i < mod->augment_size; ++i) { struct lys_node *target = mod->augment[i].target; if (!strcmp(target->module->name, "ietf-netconf-monitoring") || !strcmp(target->module->name, "ietf-netconf")) { continue; From 3b40942f59e07276d87db04938589b43c59b5845 Mon Sep 17 00:00:00 2001 From: Michal Vasko Date: Thu, 9 Aug 2018 15:14:11 +0200 Subject: [PATCH 12/49] server BUGFIX process none operation correctly Fixes #313 --- server/op_editconfig.c | 1 + 1 file changed, 1 insertion(+) diff --git a/server/op_editconfig.c b/server/op_editconfig.c index 6cbf6e4fc..0cbfbe4ea 100644 --- a/server/op_editconfig.c +++ b/server/op_editconfig.c @@ -514,6 +514,7 @@ op_editconfig(struct lyd_node *rpc, struct nc_session *ncs) break; default: /* do nothing */ + ret = 0; break; } if (valbuf) { From 00ff025a432c307bb9cd77ee7f4b284021f396c8 Mon Sep 17 00:00:00 2001 From: Michal Vasko Date: Thu, 9 Aug 2018 15:28:03 +0200 Subject: [PATCH 13/49] server TEST get rid of strncmp warning Also some minor refactoring. --- server/tests/config.h.in | 90 +++++++++++++++++++++++++++++ server/tests/test_close_session.c | 88 ---------------------------- server/tests/test_copy_config.c | 88 ---------------------------- server/tests/test_edit_get_config.c | 88 ---------------------------- server/tests/test_generic.c | 88 ---------------------------- server/tests/test_get.c | 88 ---------------------------- server/tests/test_kill.c | 88 ---------------------------- server/tests/test_notif.c | 88 ---------------------------- server/tests/test_un_lock.c | 88 ---------------------------- 9 files changed, 90 insertions(+), 704 deletions(-) diff --git a/server/tests/config.h.in b/server/tests/config.h.in index 2aa24df73..270a12b84 100644 --- a/server/tests/config.h.in +++ b/server/tests/config.h.in @@ -22,6 +22,7 @@ #include #include #include +#include /* taken from libnetconf2 session_p.h private header */ @@ -315,3 +316,92 @@ __wrap_sr_dp_get_items_subscribe(sr_session_ctx_t *session, const char *xpath, s return SR_ERR_OK; } + +/* common test functions */ +static void +test_write(int fd, const char *data, int line) +{ + int ret, written, to_write; + + written = 0; + to_write = strlen(data); + do { + ret = write(fd, data + written, to_write - written); + if (ret == -1) { + if (errno != EAGAIN) { + fprintf(stderr, "write fail (%s, line %d)\n", strerror(errno), line); + fail(); + } + usleep(100000); + ret = 0; + } + written += ret; + } while (written < to_write); + + while (((ret = write(fd, "]]>]]>", 6)) == -1) && (errno == EAGAIN)); + if (ret == -1) { + fprintf(stderr, "write fail (%s, line %d)\n", strerror(errno), line); + fail(); + } else if (ret < 6) { + fprintf(stderr, "write fail (end tag, written only %d bytes, line %d)\n", ret, line); + fail(); + } +} + +static void +test_read(int fd, const char *template, int line) +{ + char *buf, *ptr; + int ret, red, to_read; + + red = 0; + to_read = strlen(template); + buf = malloc(to_read + 1); + do { + ret = read(fd, buf + red, to_read - red); + if (ret == -1) { + if (errno != EAGAIN) { + fprintf(stderr, "read fail (%s, line %d)\n", strerror(errno), line); + fail(); + } + usleep(100000); + ret = 0; + } + red += ret; + + /* premature ending tag check */ + if ((red > 5) && !strncmp((buf + red) - 6, "]]>]]>", 6)) { + break; + } + } while (red < to_read); + buf[red] = '\0'; + + /* unify all datetimes */ + for (ptr = strstr(buf, "+02:00"); ptr; ptr = strstr(ptr + 1, "+02:00")) { + if ((ptr[-3] == ':') && (ptr[-6] == ':') && (ptr[-9] == 'T') && (ptr[-12] == '-') && (ptr[-15] == '-')) { + memcpy(ptr - 19, "0000-00-00T00:00:00", 19); + } + } + + for (red = 0; buf[red]; ++red) { + if (buf[red] != template[red]) { + fprintf(stderr, "read fail (non-matching template, line %d)\n\"%s\"(%d)\nvs. template\n\"%s\"\n", + line, buf + red, red, template + red); + fail(); + } + } + + /* read ending tag */ + while (((ret = read(fd, buf, 6)) == -1) && (errno == EAGAIN)); + if (ret == -1) { + fprintf(stderr, "read fail (%s, line %d)\n", strerror(errno), line); + fail(); + } + buf[ret] = '\0'; + if ((ret < 6) || strcmp(buf, "]]>]]>")) { + fprintf(stderr, "read fail (end tag \"%s\", line %d)\n", buf, line); + fail(); + } + + free(buf); +} diff --git a/server/tests/test_close_session.c b/server/tests/test_close_session.c index 0325d1839..2c21a8595 100644 --- a/server/tests/test_close_session.c +++ b/server/tests/test_close_session.c @@ -225,94 +225,6 @@ server_thread(void *arg) /* * TEST */ -static void -test_write(int fd, const char *data, int line) -{ - int ret, written, to_write; - - written = 0; - to_write = strlen(data); - do { - ret = write(fd, data + written, to_write - written); - if (ret == -1) { - if (errno != EAGAIN) { - fprintf(stderr, "write fail (%s, line %d)\n", strerror(errno), line); - fail(); - } - usleep(100000); - ret = 0; - } - written += ret; - } while (written < to_write); - - while (((ret = write(fd, "]]>]]>", 6)) == -1) && (errno == EAGAIN)); - if (ret == -1) { - fprintf(stderr, "write fail (%s, line %d)\n", strerror(errno), line); - fail(); - } else if (ret < 6) { - fprintf(stderr, "write fail (end tag, written only %d bytes, line %d)\n", ret, line); - fail(); - } -} - -static void -test_read(int fd, const char *template, int line) -{ - char *buf, *ptr; - int ret, red, to_read; - - red = 0; - to_read = strlen(template); - buf = malloc(to_read + 1); - do { - ret = read(fd, buf + red, to_read - red); - if (ret == -1) { - if (errno != EAGAIN) { - fprintf(stderr, "read fail (%s, line %d)\n", strerror(errno), line); - fail(); - } - usleep(100000); - ret = 0; - } - red += ret; - - /* premature ending tag check */ - if ((red > 5) && !strncmp((buf + red) - 6, "]]>]]>", 6)) { - break; - } - } while (red < to_read); - buf[red] = '\0'; - - /* unify all datetimes */ - for (ptr = strstr(buf, "+02:00"); ptr; ptr = strstr(ptr + 1, "+02:00")) { - if ((ptr[-3] == ':') && (ptr[-6] == ':') && (ptr[-9] == 'T') && (ptr[-12] == '-') && (ptr[-15] == '-')) { - strncpy(ptr - 19, "0000-00-00T00:00:00", 19); - } - } - - for (red = 0; buf[red]; ++red) { - if (buf[red] != template[red]) { - fprintf(stderr, "read fail (non-matching template, line %d)\n\"%s\"(%d)\nvs. template\n\"%s\"\n", - line, buf + red, red, template + red); - fail(); - } - } - - /* read ending tag */ - while (((ret = read(fd, buf, 6)) == -1) && (errno == EAGAIN)); - if (ret == -1) { - fprintf(stderr, "read fail (%s, line %d)\n", strerror(errno), line); - fail(); - } - buf[ret] = '\0'; - if ((ret < 6) || strcmp(buf, "]]>]]>")) { - fprintf(stderr, "read fail (end tag \"%s\", line %d)\n", buf, line); - fail(); - } - - free(buf); -} - static int np_start(void **state) { diff --git a/server/tests/test_copy_config.c b/server/tests/test_copy_config.c index 3d8cb5349..0c721f378 100644 --- a/server/tests/test_copy_config.c +++ b/server/tests/test_copy_config.c @@ -383,94 +383,6 @@ server_thread(void *arg) /* * TEST */ -static void -test_write(int fd, const char *data, int line) -{ - int ret, written, to_write; - - written = 0; - to_write = strlen(data); - do { - ret = write(fd, data + written, to_write - written); - if (ret == -1) { - if (errno != EAGAIN) { - fprintf(stderr, "write fail (%s, line %d)\n", strerror(errno), line); - fail(); - } - usleep(100000); - ret = 0; - } - written += ret; - } while (written < to_write); - - while (((ret = write(fd, "]]>]]>", 6)) == -1) && (errno == EAGAIN)); - if (ret == -1) { - fprintf(stderr, "write fail (%s, line %d)\n", strerror(errno), line); - fail(); - } else if (ret < 6) { - fprintf(stderr, "write fail (end tag, written only %d bytes, line %d)\n", ret, line); - fail(); - } -} - -static void -test_read(int fd, const char *template, int line) -{ - char *buf, *ptr; - int ret, red, to_read; - - red = 0; - to_read = strlen(template); - buf = malloc(to_read + 1); - do { - ret = read(fd, buf + red, to_read - red); - if (ret == -1) { - if (errno != EAGAIN) { - fprintf(stderr, "read fail (%s, line %d)\n", strerror(errno), line); - fail(); - } - usleep(100000); - ret = 0; - } - red += ret; - - /* premature ending tag check */ - if ((red > 5) && !strncmp((buf + red) - 6, "]]>]]>", 6)) { - break; - } - } while (red < to_read); - buf[red] = '\0'; - - /* unify all datetimes */ - for (ptr = strstr(buf, "+02:00"); ptr; ptr = strstr(ptr + 1, "+02:00")) { - if ((ptr[-3] == ':') && (ptr[-6] == ':') && (ptr[-9] == 'T') && (ptr[-12] == '-') && (ptr[-15] == '-')) { - strncpy(ptr - 19, "0000-00-00T00:00:00", 19); - } - } - - for (red = 0; buf[red]; ++red) { - if (buf[red] != template[red]) { - fprintf(stderr, "read fail (non-matching template, line %d)\n\"%s\"(%d)\nvs. template\n\"%s\"\n", - line, buf + red, red, template + red); - fail(); - } - } - - /* read ending tag */ - while (((ret = read(fd, buf, 6)) == -1) && (errno == EAGAIN)); - if (ret == -1) { - fprintf(stderr, "read fail (%s, line %d)\n", strerror(errno), line); - fail(); - } - buf[ret] = '\0'; - if ((ret < 6) || strcmp(buf, "]]>]]>")) { - fprintf(stderr, "read fail (end tag \"%s\", line %d)\n", buf, line); - fail(); - } - - free(buf); -} - static int np_start(void **state) { diff --git a/server/tests/test_edit_get_config.c b/server/tests/test_edit_get_config.c index 68a67c72d..c91925329 100755 --- a/server/tests/test_edit_get_config.c +++ b/server/tests/test_edit_get_config.c @@ -452,94 +452,6 @@ server_thread(void *arg) /* * TEST */ -static void -test_write(int fd, const char *data, int line) -{ - int ret, written, to_write; - - written = 0; - to_write = strlen(data); - do { - ret = write(fd, data + written, to_write - written); - if (ret == -1) { - if (errno != EAGAIN) { - fprintf(stderr, "write fail (%s, line %d)\n", strerror(errno), line); - fail(); - } - usleep(100000); - ret = 0; - } - written += ret; - } while (written < to_write); - - while (((ret = write(fd, "]]>]]>", 6)) == -1) && (errno == EAGAIN)); - if (ret == -1) { - fprintf(stderr, "write fail (%s, line %d)\n", strerror(errno), line); - fail(); - } else if (ret < 6) { - fprintf(stderr, "write fail (end tag, written only %d bytes, line %d)\n", ret, line); - fail(); - } -} - -static void -test_read(int fd, const char *template, int line) -{ - char *buf, *ptr; - int ret, red, to_read; - - red = 0; - to_read = strlen(template); - buf = malloc(to_read + 1); - do { - ret = read(fd, buf + red, to_read - red); - if (ret == -1) { - if (errno != EAGAIN) { - fprintf(stderr, "read fail (%s, line %d)\n", strerror(errno), line); - fail(); - } - usleep(100000); - ret = 0; - } - red += ret; - - /* premature ending tag check */ - if ((red > 5) && !strncmp((buf + red) - 6, "]]>]]>", 6)) { - break; - } - } while (red < to_read); - buf[red] = '\0'; - - /* unify all datetimes */ - for (ptr = strstr(buf, "+02:00"); ptr; ptr = strstr(ptr + 1, "+02:00")) { - if ((ptr[-3] == ':') && (ptr[-6] == ':') && (ptr[-9] == 'T') && (ptr[-12] == '-') && (ptr[-15] == '-')) { - strncpy(ptr - 19, "0000-00-00T00:00:00", 19); - } - } - - for (red = 0; buf[red]; ++red) { - if (buf[red] != template[red]) { - fprintf(stderr, "read fail (non-matching template, line %d)\n\"%s\"(%d)\nvs. template\n\"%s\"\n", - line, buf + red, red, template + red); - fail(); - } - } - - /* read ending tag */ - while (((ret = read(fd, buf, 6)) == -1) && (errno == EAGAIN)); - if (ret == -1) { - fprintf(stderr, "read fail (%s, line %d)\n", strerror(errno), line); - fail(); - } - buf[ret] = '\0'; - if ((ret < 6) || strcmp(buf, "]]>]]>")) { - fprintf(stderr, "read fail (end tag \"%s\", line %d)\n", buf, line); - fail(); - } - - free(buf); -} - static int np_start(void **state) { diff --git a/server/tests/test_generic.c b/server/tests/test_generic.c index 940944d38..62a007b05 100644 --- a/server/tests/test_generic.c +++ b/server/tests/test_generic.c @@ -280,94 +280,6 @@ server_thread(void *arg) /* * TEST */ -static void -test_write(int fd, const char *data, int line) -{ - int ret, written, to_write; - - written = 0; - to_write = strlen(data); - do { - ret = write(fd, data + written, to_write - written); - if (ret == -1) { - if (errno != EAGAIN) { - fprintf(stderr, "write fail (%s, line %d)\n", strerror(errno), line); - fail(); - } - usleep(100000); - ret = 0; - } - written += ret; - } while (written < to_write); - - while (((ret = write(fd, "]]>]]>", 6)) == -1) && (errno == EAGAIN)); - if (ret == -1) { - fprintf(stderr, "write fail (%s, line %d)\n", strerror(errno), line); - fail(); - } else if (ret < 6) { - fprintf(stderr, "write fail (end tag, written only %d bytes, line %d)\n", ret, line); - fail(); - } -} - -static void -test_read(int fd, const char *template, int line) -{ - char *buf, *ptr; - int ret, red, to_read; - - red = 0; - to_read = strlen(template); - buf = malloc(to_read + 1); - do { - ret = read(fd, buf + red, to_read - red); - if (ret == -1) { - if (errno != EAGAIN) { - fprintf(stderr, "read fail (%s, line %d)\n", strerror(errno), line); - fail(); - } - usleep(100000); - ret = 0; - } - red += ret; - - /* premature ending tag check */ - if ((red > 5) && !strncmp((buf + red) - 6, "]]>]]>", 6)) { - break; - } - } while (red < to_read); - buf[red] = '\0'; - - /* unify all datetimes */ - for (ptr = strchr(buf, '+'); ptr; ptr = strchr(ptr + 1, '+')) { - if ((ptr[-3] == ':') && (ptr[-6] == ':') && (ptr[-9] == 'T') && (ptr[-12] == '-') && (ptr[-15] == '-')) { - strncpy(ptr - 19, "0000-00-00T00:00:00+00:00", 25); - } - } - - for (red = 0; buf[red]; ++red) { - if (buf[red] != template[red]) { - fprintf(stderr, "read fail (non-matching template, line %d)\n\"%s\"(%d)\nvs. template\n\"%s\"\n", - line, buf + red, red, template + red); - fail(); - } - } - - /* read ending tag */ - while (((ret = read(fd, buf, 6)) == -1) && (errno == EAGAIN)); - if (ret == -1) { - fprintf(stderr, "read fail (%s, line %d)\n", strerror(errno), line); - fail(); - } - buf[ret] = '\0'; - if ((ret < 6) || strcmp(buf, "]]>]]>")) { - fprintf(stderr, "read fail (end tag \"%s\", line %d)\n", buf, line); - fail(); - } - - free(buf); -} - static int np_start(void **state) { diff --git a/server/tests/test_get.c b/server/tests/test_get.c index 28e33f2be..f2ba2d622 100644 --- a/server/tests/test_get.c +++ b/server/tests/test_get.c @@ -322,94 +322,6 @@ server_thread(void *arg) /* * TEST */ -static void -test_write(int fd, const char *data, int line) -{ - int ret, written, to_write; - - written = 0; - to_write = strlen(data); - do { - ret = write(fd, data + written, to_write - written); - if (ret == -1) { - if (errno != EAGAIN) { - fprintf(stderr, "write fail (%s, line %d)\n", strerror(errno), line); - fail(); - } - usleep(100000); - ret = 0; - } - written += ret; - } while (written < to_write); - - while (((ret = write(fd, "]]>]]>", 6)) == -1) && (errno == EAGAIN)); - if (ret == -1) { - fprintf(stderr, "write fail (%s, line %d)\n", strerror(errno), line); - fail(); - } else if (ret < 6) { - fprintf(stderr, "write fail (end tag, written only %d bytes, line %d)\n", ret, line); - fail(); - } -} - -static void -test_read(int fd, const char *template, int line) -{ - char *buf, *ptr; - int ret, red, to_read; - - red = 0; - to_read = strlen(template); - buf = malloc(to_read + 1); - do { - ret = read(fd, buf + red, to_read - red); - if (ret == -1) { - if (errno != EAGAIN) { - fprintf(stderr, "read fail (%s, line %d)\n", strerror(errno), line); - fail(); - } - usleep(100000); - ret = 0; - } - red += ret; - - /* premature ending tag check */ - if ((red > 5) && !strncmp((buf + red) - 6, "]]>]]>", 6)) { - break; - } - } while (red < to_read); - buf[red] = '\0'; - - /* unify all datetimes */ - for (ptr = strchr(buf, '+'); ptr; ptr = strchr(ptr + 1, '+')) { - if ((ptr[-3] == ':') && (ptr[-6] == ':') && (ptr[-9] == 'T') && (ptr[-12] == '-') && (ptr[-15] == '-')) { - strncpy(ptr - 19, "0000-00-00T00:00:00+00:00", 25); - } - } - - for (red = 0; buf[red]; ++red) { - if (buf[red] != template[red]) { - fprintf(stderr, "read fail (non-matching template, line %d)\n\"%s\"(%d)\nvs. template\n\"%s\"\n", - line, buf + red, red, template + red); - fail(); - } - } - - /* read ending tag */ - while (((ret = read(fd, buf, 6)) == -1) && (errno == EAGAIN)); - if (ret == -1) { - fprintf(stderr, "read fail (%s, line %d)\n", strerror(errno), line); - fail(); - } - buf[ret] = '\0'; - if ((ret < 6) || strcmp(buf, "]]>]]>")) { - fprintf(stderr, "read fail (end tag \"%s\", line %d)\n", buf, line); - fail(); - } - - free(buf); -} - static int np_start(void **state) { diff --git a/server/tests/test_kill.c b/server/tests/test_kill.c index 9ced9e1a8..575011fbc 100644 --- a/server/tests/test_kill.c +++ b/server/tests/test_kill.c @@ -236,94 +236,6 @@ server_thread(void *arg) /* * TEST */ -static void -test_write(int fd, const char *data, int line) -{ - int ret, written, to_write; - - written = 0; - to_write = strlen(data); - do { - ret = write(fd, data + written, to_write - written); - if (ret == -1) { - if (errno != EAGAIN) { - fprintf(stderr, "write fail (%s, line %d)\n", strerror(errno), line); - fail(); - } - usleep(100000); - ret = 0; - } - written += ret; - } while (written < to_write); - - while (((ret = write(fd, "]]>]]>", 6)) == -1) && (errno == EAGAIN)); - if (ret == -1) { - fprintf(stderr, "write fail (%s, line %d)\n", strerror(errno), line); - fail(); - } else if (ret < 6) { - fprintf(stderr, "write fail (end tag, written only %d bytes, line %d)\n", ret, line); - fail(); - } -} - -static void -test_read(int fd, const char *template, int line) -{ - char *buf, *ptr; - int ret, red, to_read; - - red = 0; - to_read = strlen(template); - buf = malloc(to_read + 1); - do { - ret = read(fd, buf + red, to_read - red); - if (ret == -1) { - if (errno != EAGAIN) { - fprintf(stderr, "read fail (%s, line %d)\n", strerror(errno), line); - fail(); - } - usleep(100000); - ret = 0; - } - red += ret; - - /* premature ending tag check */ - if ((red > 5) && !strncmp((buf + red) - 6, "]]>]]>", 6)) { - break; - } - } while (red < to_read); - buf[red] = '\0'; - - /* unify all datetimes */ - for (ptr = strstr(buf, "+02:00"); ptr; ptr = strstr(ptr + 1, "+02:00")) { - if ((ptr[-3] == ':') && (ptr[-6] == ':') && (ptr[-9] == 'T') && (ptr[-12] == '-') && (ptr[-15] == '-')) { - strncpy(ptr - 19, "0000-00-00T00:00:00", 19); - } - } - - for (red = 0; buf[red]; ++red) { - if (buf[red] != template[red]) { - fprintf(stderr, "read fail (non-matching template, line %d)\n\"%s\"(%d)\nvs. template\n\"%s\"\n", - line, buf + red, red, template + red); - fail(); - } - } - - /* read ending tag */ - while (((ret = read(fd, buf, 6)) == -1) && (errno == EAGAIN)); - if (ret == -1) { - fprintf(stderr, "read fail (%s, line %d)\n", strerror(errno), line); - fail(); - } - buf[ret] = '\0'; - if ((ret < 6) || strcmp(buf, "]]>]]>")) { - fprintf(stderr, "read fail (end tag \"%s\", line %d)\n", buf, line); - fail(); - } - - free(buf); -} - static int np_start(void **state) { diff --git a/server/tests/test_notif.c b/server/tests/test_notif.c index a23ec923c..990f114aa 100644 --- a/server/tests/test_notif.c +++ b/server/tests/test_notif.c @@ -285,94 +285,6 @@ server_thread(void *arg) /* * TEST */ -static void -test_write(int fd, const char *data, int line) -{ - int ret, written, to_write; - - written = 0; - to_write = strlen(data); - do { - ret = write(fd, data + written, to_write - written); - if (ret == -1) { - if (errno != EAGAIN) { - fprintf(stderr, "write fail (%s, line %d)\n", strerror(errno), line); - fail(); - } - usleep(100000); - ret = 0; - } - written += ret; - } while (written < to_write); - - while (((ret = write(fd, "]]>]]>", 6)) == -1) && (errno == EAGAIN)); - if (ret == -1) { - fprintf(stderr, "write fail (%s, line %d)\n", strerror(errno), line); - fail(); - } else if (ret < 6) { - fprintf(stderr, "write fail (end tag, written only %d bytes, line %d)\n", ret, line); - fail(); - } -} - -static void -test_read(int fd, const char *template, int line) -{ - char *buf, *ptr; - int ret, red, to_read; - - red = 0; - to_read = strlen(template); - buf = malloc(to_read + 1); - do { - ret = read(fd, buf + red, to_read - red); - if (ret == -1) { - if (errno != EAGAIN) { - fprintf(stderr, "read fail (%s, line %d)\n", strerror(errno), line); - fail(); - } - usleep(100000); - ret = 0; - } - red += ret; - - /* premature ending tag check */ - if ((red > 5) && !strncmp((buf + red) - 6, "]]>]]>", 6)) { - break; - } - } while (red < to_read); - buf[red] = '\0'; - - /* unify all datetimes */ - for (ptr = strchr(buf, '-'); ptr; ptr = strchr(ptr + 1, '-')) { - if ((ptr[3] == '-') && (ptr[6] == 'T') && (ptr[9] == ':') && (ptr[12] == ':')) { - strncpy(ptr - 4, "0000-00-00T00:00:00", 19); - } - } - - for (red = 0; buf[red]; ++red) { - if (buf[red] != template[red]) { - fprintf(stderr, "read fail (non-matching template, line %d)\n\"%s\"(%d)\nvs. template\n\"%s\"\n", - line, buf + red, red, template + red); - fail(); - } - } - - /* read ending tag */ - while (((ret = read(fd, buf, 6)) == -1) && (errno == EAGAIN)); - if (ret == -1) { - fprintf(stderr, "read fail (%s, line %d)\n", strerror(errno), line); - fail(); - } - buf[ret] = '\0'; - if ((ret < 6) || strcmp(buf, "]]>]]>")) { - fprintf(stderr, "read fail (end tag \"%s\", line %d)\n", buf, line); - fail(); - } - - free(buf); -} - static int np_start(void **state) { diff --git a/server/tests/test_un_lock.c b/server/tests/test_un_lock.c index 58081ca24..d3b4f86bd 100644 --- a/server/tests/test_un_lock.c +++ b/server/tests/test_un_lock.c @@ -298,94 +298,6 @@ server_thread(void *arg) /* * TEST */ -static void -test_write(int fd, const char *data, int line) -{ - int ret, written, to_write; - - written = 0; - to_write = strlen(data); - do { - ret = write(fd, data + written, to_write - written); - if (ret == -1) { - if (errno != EAGAIN) { - fprintf(stderr, "write fail (%s, line %d)\n", strerror(errno), line); - fail(); - } - usleep(100000); - ret = 0; - } - written += ret; - } while (written < to_write); - - while (((ret = write(fd, "]]>]]>", 6)) == -1) && (errno == EAGAIN)); - if (ret == -1) { - fprintf(stderr, "write fail (%s, line %d)\n", strerror(errno), line); - fail(); - } else if (ret < 6) { - fprintf(stderr, "write fail (end tag, written only %d bytes, line %d)\n", ret, line); - fail(); - } -} - -static void -test_read(int fd, const char *template, int line) -{ - char *buf, *ptr; - int ret, red, to_read; - - red = 0; - to_read = strlen(template); - buf = malloc(to_read + 1); - do { - ret = read(fd, buf + red, to_read - red); - if (ret == -1) { - if (errno != EAGAIN) { - fprintf(stderr, "read fail (%s, line %d)\n", strerror(errno), line); - fail(); - } - usleep(100000); - ret = 0; - } - red += ret; - - /* premature ending tag check */ - if ((red > 5) && !strncmp((buf + red) - 6, "]]>]]>", 6)) { - break; - } - } while (red < to_read); - buf[red] = '\0'; - - /* unify all datetimes */ - for (ptr = strstr(buf, "+02:00"); ptr; ptr = strstr(ptr + 1, "+02:00")) { - if ((ptr[-3] == ':') && (ptr[-6] == ':') && (ptr[-9] == 'T') && (ptr[-12] == '-') && (ptr[-15] == '-')) { - strncpy(ptr - 19, "0000-00-00T00:00:00", 19); - } - } - - for (red = 0; buf[red]; ++red) { - if (buf[red] != template[red]) { - fprintf(stderr, "read fail (non-matching template, line %d)\n\"%s\"(%d)\nvs. template\n\"%s\"\n", - line, buf + red, red, template + red); - fail(); - } - } - - /* read ending tag */ - while (((ret = read(fd, buf, 6)) == -1) && (errno == EAGAIN)); - if (ret == -1) { - fprintf(stderr, "read fail (%s, line %d)\n", strerror(errno), line); - fail(); - } - buf[ret] = '\0'; - if ((ret < 6) || strcmp(buf, "]]>]]>")) { - fprintf(stderr, "read fail (end tag \"%s\", line %d)\n", buf, line); - fail(); - } - - free(buf); -} - static int np_start(void **state) { From 7e73237f3d6d1ce6e7e6cf101a79ddbcd0f0f043 Mon Sep 17 00:00:00 2001 From: Michal Vasko Date: Thu, 9 Aug 2018 15:30:34 +0200 Subject: [PATCH 14/49] server VERSION bump to version 0.6.2 --- server/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/CMakeLists.txt b/server/CMakeLists.txt index 26ce32361..782f81ad6 100755 --- a/server/CMakeLists.txt +++ b/server/CMakeLists.txt @@ -13,7 +13,7 @@ if(NOT UNIX) endif() # set version -set(NP2SRV_VERSION 0.6.1) +set(NP2SRV_VERSION 0.6.2) # set default build type if not specified by user if(NOT CMAKE_BUILD_TYPE) From 4dd0bc20aff2dbf5c5fde0b2d63904bc5de829cd Mon Sep 17 00:00:00 2001 From: Michal Vasko Date: Thu, 9 Aug 2018 15:50:01 +0200 Subject: [PATCH 15/49] fixup! server TEST get rid of strncmp warning --- server/tests/config.h.in | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/server/tests/config.h.in b/server/tests/config.h.in index 270a12b84..2ffab232f 100644 --- a/server/tests/config.h.in +++ b/server/tests/config.h.in @@ -382,6 +382,12 @@ test_read(int fd, const char *template, int line) memcpy(ptr - 19, "0000-00-00T00:00:00", 19); } } + for (ptr = strchr(buf, '-'); ptr; ptr = strchr(ptr + 1, '-')) { + if ((ptr[3] == '-') && (ptr[6] == 'T') && (ptr[9] == ':') && (ptr[12] == ':')) { + memcpy(ptr - 4, "0000-00-00T00:00:00", 19); + } + } + for (red = 0; buf[red]; ++red) { if (buf[red] != template[red]) { From 21bc3ed4b8fd460fce3efc12d16f54cd3c861b18 Mon Sep 17 00:00:00 2001 From: Andrew Langefeld Date: Fri, 10 Aug 2018 13:39:49 -0500 Subject: [PATCH 16/49] Cache ly_ctx_info to improve get performance Fixes #304 This change results in a 3x to 4x speedup for unfiltered get requests (or requests for the ietf-yang-library contents). We now cache the results of the `ly_ctx_info(np2srv.ly_ctx)` call and save it in the `np2srv` global, to avoid repeating the call when it would just return the same data. We update the cached value when necessary, namely, when a module or feature is changed. Because the contents of the cache should change relatively infrequently, it seems worth it to pay the cost early instead of for every request. --- server/common.h | 1 + server/main.c | 21 +++++++++++++++++++-- 2 files changed, 20 insertions(+), 2 deletions(-) diff --git a/server/common.h b/server/common.h index c420caba6..02c9b8dba 100644 --- a/server/common.h +++ b/server/common.h @@ -49,6 +49,7 @@ struct np2srv { pthread_t workers[NP2SRV_THREAD_COUNT]; /**< worker threads handling sessions */ struct ly_ctx *ly_ctx; /**< libyang's context */ + struct lyd_node *ly_ctx_info_cache; /** < a cache of calling ly_ctx_info on the ly_ctx */ pthread_rwlock_t ly_ctx_lock; /**< libyang's context rwlock */ }; extern struct np2srv np2srv; diff --git a/server/main.c b/server/main.c index ca95be317..ffd8f0268 100644 --- a/server/main.c +++ b/server/main.c @@ -434,6 +434,15 @@ np2srv_create_capab(const struct lys_module *mod) return cpb; } +static void +np2srv_update_ly_ctx_info_cache() +{ + if (np2srv.ly_ctx_info_cache) { + lyd_free_withsiblings(np2srv.ly_ctx_info_cache); + } + np2srv.ly_ctx_info_cache = ly_ctx_info(np2srv.ly_ctx); +} + static void np2srv_module_install_clb(const char *module_name, const char *revision, sr_module_state_t state, void *UNUSED(private_ctx)) { @@ -516,6 +525,7 @@ np2srv_module_install_clb(const char *module_name, const char *revision, sr_modu free(cpb); } + np2srv_update_ly_ctx_info_cache(); /* unlock libyang context */ pthread_rwlock_unlock(&np2srv.ly_ctx_lock); @@ -550,6 +560,7 @@ np2srv_feature_change_clb(const char *module_name, const char *feature_name, boo lys_features_disable(mod, feature_name); } cpb = np2srv_create_capab(mod); + np2srv_update_ly_ctx_info_cache(); pthread_rwlock_unlock(&np2srv.ly_ctx_lock); np2srv_send_capab_change_notif(NULL, NULL, cpb); @@ -562,6 +573,7 @@ np2srv_state_data_clb(const char *xpath, sr_val_t **values, size_t *values_cnt, struct lyd_node *data = NULL, *node, *iter; struct ly_set *set = NULL; uint32_t i, j; + bool should_free_data = true; int ret = SR_ERR_OK; if (!strncmp(xpath, "/ietf-netconf-monitoring:", 25)) { @@ -569,7 +581,8 @@ np2srv_state_data_clb(const char *xpath, sr_val_t **values, size_t *values_cnt, } else if (!strncmp(xpath, "/nc-notifications:", 18)) { data = ntf_get_data(); } else if (!strncmp(xpath, "/ietf-yang-library:", 19)) { - data = ly_ctx_info(np2srv.ly_ctx); + data = np2srv.ly_ctx_info_cache; + should_free_data = false; } else { ret = SR_ERR_OPERATION_FAILED; goto cleanup; @@ -636,7 +649,9 @@ np2srv_state_data_clb(const char *xpath, sr_val_t **values, size_t *values_cnt, cleanup: ly_set_free(set); - lyd_free_withsiblings(data); + if (should_free_data) { + lyd_free_withsiblings(data); + } if (ret != SR_ERR_OK) { sr_free_values(*values, *values_cnt); *values_cnt = 0; @@ -1102,6 +1117,8 @@ np2srv_init_schemas(void) goto error; } + np2srv_update_ly_ctx_info_cache(); + /* debug - list schemas struct lyd_node *ylib = ly_ctx_info(np2srv.ly_ctx); lyd_print_file(stdout, ylib, LYD_JSON, LYP_WITHSIBLINGS); From 6ab061774582d72ed478c3364aedd605be9e104d Mon Sep 17 00:00:00 2001 From: Michal Vasko Date: Mon, 13 Aug 2018 15:29:06 +0200 Subject: [PATCH 17/49] server OPTIMIZE cache creating data tree from sysrepo -> Optimize transforming sr_val_t to struct lyd_node. --- server/op_get_config.c | 5 +- server/op_notifications.c | 6 +- server/operations.c | 294 ++++++++++++++++++++++++++++++++++++-- server/operations.h | 16 ++- 4 files changed, 307 insertions(+), 14 deletions(-) diff --git a/server/op_get_config.c b/server/op_get_config.c index 40987c238..daa677bb0 100644 --- a/server/op_get_config.c +++ b/server/op_get_config.c @@ -36,6 +36,7 @@ opget_build_subtree_from_sysrepo(sr_session_ctx_t *srs, struct lyd_node **root, sr_val_t *value; sr_val_iter_t *sriter; struct lyd_node *node; + struct sr2ly_cache cache; char *full_subtree_xpath = NULL; int rc; @@ -44,6 +45,7 @@ opget_build_subtree_from_sysrepo(sr_session_ctx_t *srs, struct lyd_node **root, return -1; } + memset(&cache, 0, sizeof cache); np2srv_sr_session_refresh(srs, NULL); rc = np2srv_sr_get_items_iter(srs, full_subtree_xpath, &sriter, NULL); @@ -56,7 +58,7 @@ opget_build_subtree_from_sysrepo(sr_session_ctx_t *srs, struct lyd_node **root, } while ((!np2srv_sr_get_item_next(srs, sriter, &value, NULL))) { - if (op_sr_val_to_lyd_node(*root, value, &node)) { + if (op_sr2ly(*root, value, &node, &cache)) { sr_free_val(value); sr_free_val_iter(sriter); return -1; @@ -69,6 +71,7 @@ opget_build_subtree_from_sysrepo(sr_session_ctx_t *srs, struct lyd_node **root, } sr_free_val_iter(sriter); + op_sr2ly_free_cache(&cache); return 0; } diff --git a/server/op_notifications.c b/server/op_notifications.c index 197ac579f..801201bde 100644 --- a/server/op_notifications.c +++ b/server/op_notifications.c @@ -190,9 +190,12 @@ np2srv_ntf_clb(const sr_ev_notif_type_t notif_type, const char *xpath, const sr_ { struct np_subscriber *subscriber = (struct np_subscriber *)private_ctx; struct lyd_node *ntf = NULL, *node; + struct sr2ly_cache cache; size_t i; const char *ntf_type_str = NULL; + memset(&cache, 0, sizeof cache); + switch (notif_type) { case SR_EV_NOTIF_T_REALTIME: ntf_type_str = "realtime"; @@ -219,7 +222,7 @@ np2srv_ntf_clb(const sr_ev_notif_type_t notif_type, const char *xpath, const sr_ } for (i = 0; i < val_cnt; i++) { - if (op_sr_val_to_lyd_node(ntf, &vals[i], &node)) { + if (op_sr2ly(ntf, &vals[i], &node, &cache)) { ERR("Creating notification (%s) data (%s) failed.", xpath, vals[i].xpath); goto error; } @@ -230,6 +233,7 @@ np2srv_ntf_clb(const sr_ev_notif_type_t notif_type, const char *xpath, const sr_ np2srv_ntf_send(subscriber, ntf, timestamp, notif_type); error: + op_sr2ly_free_cache(&cache); lyd_free(ntf); } diff --git a/server/operations.c b/server/operations.c index 247d322d3..8426baec2 100644 --- a/server/operations.c +++ b/server/operations.c @@ -2630,25 +2630,299 @@ op_filter_create(struct lyd_node *filter_node, char ***filters, int *filter_coun return 0; } +static const char * +op_sr2ly_parse_node(const char *xpath, const char **mod, int *mod_len, const char **name, int *name_len, + const char **pred, int *pred_len) +{ + if (xpath[0] != '/') { + return NULL; + } + + /* parse module name */ + *mod = xpath + 1; + *mod_len = 0; + while (((*mod)[*mod_len] != ':') && ((*mod)[*mod_len] != '/') && ((*mod)[*mod_len] != '[') + && ((*mod)[*mod_len] != '\0')) { + ++(*mod_len); + } + if ((*mod)[*mod_len] != ':') { + /* no module name, this is the node name */ + *name = *mod; + *mod = NULL; + *name_len = *mod_len; + *mod_len = 0; + } else { + /* parse node name */ + *name = *mod + *mod_len + 1; + *name_len = 0; + while (((*name)[*name_len] != '/') && ((*name)[*name_len] != '[') && ((*name)[*name_len] != '\0')) { + ++(*name_len); + } + } + + /* parse all predicates */ + if ((*name)[*name_len] == '[') { + *pred = *name + *name_len; + *pred_len = 0; + do { + while ((*pred)[*pred_len] != ']') { + ++(*pred_len); + } + ++(*pred_len); + } while ((*pred)[*pred_len] == '['); + + xpath = *pred + *pred_len; + } else { + *pred = NULL; + *pred_len = 0; + + xpath = *name + *name_len; + } + + return xpath; +} + +static void +op_sr2ly_add_cache(struct sr2ly_cache *cache, const char *mod, int mod_len, const char *name, int name_len, + const char *pred, int pred_len) +{ + if (cache->used == cache->size) { + ++cache->size; + cache->items = realloc(cache->items, cache->size * sizeof *cache->items); + } + ++cache->used; + cache->items[cache->used - 1].node = NULL; + cache->items[cache->used - 1].mod = mod ? strndup(mod, mod_len) : NULL; + cache->items[cache->used - 1].name = strndup(name, name_len); + cache->items[cache->used - 1].pred = pred ? strndup(pred, pred_len) : NULL; +} + +static int +op_sr2ly_get_cache_parent(const char *xpath, struct sr2ly_cache *cache, struct lyd_node *root, int *new_node_i) +{ + const char *mod, *name, *pred; + char *path; + const struct lys_module *module; + size_t i; + int mod_len, name_len, pred_len; + struct ly_set *set; + + /* look for the parent in the cache */ + for (i = 0; i < cache->used; ++i) { + xpath = op_sr2ly_parse_node(xpath, &mod, &mod_len, &name, &name_len, &pred, &pred_len); + + /* node name */ + if (strncmp(name, cache->items[i].name, name_len) || cache->items[i].name[name_len]) { + break; + } + + /* node module name */ + if ((mod && !cache->items[i].mod) || (!mod && cache->items[i].mod)) { + break; + } + if (mod && (strncmp(mod, cache->items[i].mod, mod_len) || cache->items[i].mod[mod_len])) { + break; + } + + /* node predicates */ + if ((pred && !cache->items[i].pred) || (!pred && cache->items[i].pred)) { + /* should not ever happen, actually */ + break; + } + if (pred && (strncmp(pred, cache->items[i].pred, pred_len) || cache->items[i].pred[pred_len])) { + break; + } + + /* we have matched, try to find the next match */ + } + + if (i < cache->used) { + /* remove unused items */ + while (cache->used > i) { + free(cache->items[cache->used - 1].name); + free(cache->items[cache->used - 1].mod); + free(cache->items[cache->used - 1].pred); + + --cache->used; + } + } else if (xpath[0]) { + /* all parsed nodes matched, get the next one */ + xpath = op_sr2ly_parse_node(xpath, &mod, &mod_len, &name, &name_len, &pred, &pred_len); + } + + /* some parents did not match, either the parent does not exist or is just not cached */ + while (xpath[0]) { + module = NULL; + set = NULL; + if (root) { + /* build path to find */ + asprintf(&path, "%s%.*s%s%.*s%.*s", i ? "" : "/", mod_len, mod ? mod : "", mod ? ":" : "", name_len, name, + pred_len, pred ? pred : ""); + if (cache->used) { + /* relative from parent */ + set = lyd_find_path(cache->items[cache->used - 1].node, path); + } else { + /* top-level node */ + set = lyd_find_path(root, path); + } + free(path); + + if (!set || (set->number > 1)) { + ly_set_free(set); + return -1; + } + } + + /* create new item for this parent */ + op_sr2ly_add_cache(cache, mod, mod_len, name, name_len, pred, pred_len); + + if (!root || !set->number) { + /* create the parent node first, it does not exist */ + if (cache->items[cache->used - 1].mod) { + module = ly_ctx_get_module(np2srv.ly_ctx, cache->items[cache->used - 1].mod, NULL, 1); + if (!module) { + ly_set_free(set); + return -1; + } + } + + /* it is a parent, must be list or container (or action, notif, whatever) */ + if (cache->used > 1) { + /* inner node */ + cache->items[cache->used - 1].node = lyd_new(cache->items[cache->used - 2].node, module, cache->items[cache->used - 1].name); + } else { + /* top-level node */ + cache->items[cache->used - 1].node = lyd_new(NULL, module, cache->items[cache->used - 1].name); + } + if (!cache->items[cache->used - 1].node) { + ly_set_free(set); + return -1; + } + if (*new_node_i == -1) { + /* first created node */ + *new_node_i = cache->used - 1; + } + } else { + /* the parent exists, cache it */ + cache->items[cache->used - 1].node = set->set.d[0]; + } + ly_set_free(set); + + /* parse next node */ + xpath = op_sr2ly_parse_node(xpath, &mod, &mod_len, &name, &name_len, &pred, &pred_len); + } + + /* create item for the newly created node and return its index */ + op_sr2ly_add_cache(cache, mod, mod_len, name, name_len, pred, pred_len); + return 0; +} + int -op_sr_val_to_lyd_node(struct lyd_node *root, const sr_val_t *sr_val, struct lyd_node **new_node) +op_sr2ly(struct lyd_node *root, const sr_val_t *sr_val, struct lyd_node **new_node, struct sr2ly_cache *cache) { char numstr[22]; - char *str; + char *val_str; + int new_node_i = -1; + struct lyd_node *parent = NULL, *node; + const struct lys_module *mod = NULL; - str = op_get_srval(np2srv.ly_ctx, sr_val, numstr); - if (!str) { - str = ""; + /* last used index in cache is ours, we can learn everything based on that */ + if (op_sr2ly_get_cache_parent(sr_val->xpath, cache, root, &new_node_i)) { + return -1; } - ly_errno = LY_SUCCESS; + /* get module */ + if (cache->items[cache->used - 1].mod) { + mod = ly_ctx_get_module(np2srv.ly_ctx, cache->items[cache->used - 1].mod, NULL, 1); + if (!mod) { + return -1; + } + } + + /* get parent */ + if (cache->used > 1) { + parent = cache->items[cache->used - 2].node; + } - *new_node = lyd_new_path(root, np2srv.ly_ctx, sr_val->xpath, str, - (sr_val->type == SR_ANYXML_T || sr_val->type == SR_ANYDATA_T) ? LYD_ANYDATA_SXML : 0, - LYD_PATH_OPT_UPDATE | (sr_val->dflt ? LYD_PATH_OPT_DFLT : 0)); - if (ly_errno) { + /* create the new node */ + switch (sr_val->type) { + case SR_STRING_T: + case SR_BINARY_T: + case SR_BITS_T: + case SR_ENUM_T: + case SR_IDENTITYREF_T: + case SR_INSTANCEID_T: + case SR_LEAF_EMPTY_T: + case SR_BOOL_T: + case SR_DECIMAL64_T: + case SR_UINT8_T: + case SR_UINT16_T: + case SR_UINT32_T: + case SR_UINT64_T: + case SR_INT8_T: + case SR_INT16_T: + case SR_INT32_T: + case SR_INT64_T: + val_str = op_get_srval(np2srv.ly_ctx, sr_val, numstr); + node = lyd_new_leaf(parent, mod, cache->items[cache->used - 1].name, val_str); + break; + case SR_ANYDATA_T: + case SR_ANYXML_T: + val_str = op_get_srval(np2srv.ly_ctx, sr_val, numstr); + node = lyd_new_anydata(parent, mod, cache->items[cache->used - 1].name, val_str, LYD_ANYDATA_SXML); + break; + case SR_LIST_T: + case SR_CONTAINER_T: + case SR_CONTAINER_PRESENCE_T: + node = lyd_new(parent, mod, cache->items[cache->used - 1].name); + break; + default: return -1; } + if (!node) { + return -1; + } + + /* inherit dflt flag */ + node->dflt = sr_val->dflt; + /* insert into data tree if required */ + if (!parent && root) { + if (lyd_insert_after(root->prev, node)) { + return -1; + } + } + + if (new_node_i == -1) { + /* this is the first created node, return it */ + *new_node = node; + } else { + /* return the parent created before */ + *new_node = cache->items[new_node_i].node; + } + + if (node->schema->nodetype & (LYS_CONTAINER | LYS_LIST)) { + /* store in cache */ + cache->items[cache->used - 1].node = node; + } else { + /* useless to store ending nodes in cache */ + free(cache->items[cache->used - 1].mod); + free(cache->items[cache->used - 1].name); + free(cache->items[cache->used - 1].pred); + --cache->used; + } return 0; } + +void +op_sr2ly_free_cache(struct sr2ly_cache *cache) +{ + size_t i; + + for (i = 0; i < cache->used; ++i) { + free(cache->items[i].mod); + free(cache->items[i].name); + free(cache->items[i].pred); + } + free(cache->items); +} diff --git a/server/operations.h b/server/operations.h index cf0ca2c08..824ef128a 100644 --- a/server/operations.h +++ b/server/operations.h @@ -141,7 +141,20 @@ struct nc_server_reply *op_build_err_nacm(struct nc_server_reply *ereply); int op_filter_get_tree_from_data(struct lyd_node **root, struct lyd_node *data, const char *subtree_path); int op_filter_xpath_add_filter(char *new_filter, char ***filters, int *filter_count); int op_filter_create(struct lyd_node *filter_node, char ***filters, int *filter_count); -int op_sr_val_to_lyd_node(struct lyd_node *root, const sr_val_t *sr_val, struct lyd_node **new_node); + +struct sr2ly_cache { + struct { + struct lyd_node *node; + char *name; + char *mod; + char *pred; + } *items; + size_t used; + size_t size; +}; + +void op_sr2ly_free_cache(struct sr2ly_cache *cache); +int op_sr2ly(struct lyd_node *root, const sr_val_t *sr_val, struct lyd_node **new_node, struct sr2ly_cache *cache); struct nc_server_reply *op_get(struct lyd_node *rpc, struct nc_session *ncs); struct nc_server_reply *op_lock(struct lyd_node *rpc, struct nc_session *ncs); @@ -160,5 +173,4 @@ void op_ntf_unsubscribe(struct nc_session *session); void op_ntf_yang_lib_change(const struct lyd_node *ylib_info); struct lyd_node *ntf_get_data(void); - #endif /* NP2SRV_OPERATIONS_H_ */ From fbe9ab6c5a27f2c9fb630fac6bc3a2e9d57bd42f Mon Sep 17 00:00:00 2001 From: Michal Vasko Date: Mon, 13 Aug 2018 15:30:10 +0200 Subject: [PATCH 18/49] server VERSION bump to version 0.6.3 --- server/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/CMakeLists.txt b/server/CMakeLists.txt index 782f81ad6..a67d68126 100755 --- a/server/CMakeLists.txt +++ b/server/CMakeLists.txt @@ -13,7 +13,7 @@ if(NOT UNIX) endif() # set version -set(NP2SRV_VERSION 0.6.2) +set(NP2SRV_VERSION 0.6.3) # set default build type if not specified by user if(NOT CMAKE_BUILD_TYPE) From eff0b2d4dcc859263871d14fc129c3c8be3c0afe Mon Sep 17 00:00:00 2001 From: Andrew Langefeld Date: Mon, 13 Aug 2018 21:47:48 -0500 Subject: [PATCH 19/49] Make cache optional, and use module-set-id Address feedback from pull request. Flag the new functionality with `ENABLE_LY_CTX_INFO_CACHE` (which defaults to `ON`), and use module-set-id to determine if a cache update is needed. --- server/CMakeLists.txt | 5 +++++ server/common.h | 5 ++++- server/config.h.in | 4 ++++ server/main.c | 18 +++++++++++++----- 4 files changed, 26 insertions(+), 6 deletions(-) mode change 100644 => 100755 server/config.h.in diff --git a/server/CMakeLists.txt b/server/CMakeLists.txt index 782f81ad6..964d9e28e 100755 --- a/server/CMakeLists.txt +++ b/server/CMakeLists.txt @@ -36,6 +36,11 @@ option(ENABLE_CONFIGURATION "Enable server configuration" ON) set(THREAD_COUNT 5 CACHE STRING "Number of threads accepting new sessions and handling requests") set(DEFAULT_HOST_KEY "/etc/ssh/ssh_host_rsa_key" CACHE STRING "Default server host key (used only if configuration is disabled)") +option(ENABLE_LY_CTX_INFO_CACHE "Enable caching the ly_ctx_info() result; reduces processing at the cost of increased memory usage." ON) +if(ENABLE_LY_CTX_INFO_CACHE) + set(NP2SRV_ENABLED_LY_CTX_INFO_CACHE 1) +endif() + # set prefix for the PID file if (NOT PIDFILE_PREFIX) set(PIDFILE_PREFIX "/var/run") diff --git a/server/common.h b/server/common.h index 02c9b8dba..a946407fd 100644 --- a/server/common.h +++ b/server/common.h @@ -49,7 +49,10 @@ struct np2srv { pthread_t workers[NP2SRV_THREAD_COUNT]; /**< worker threads handling sessions */ struct ly_ctx *ly_ctx; /**< libyang's context */ - struct lyd_node *ly_ctx_info_cache; /** < a cache of calling ly_ctx_info on the ly_ctx */ +#ifdef NP2SRV_ENABLED_LY_CTX_INFO_CACHE + uint16_t cached_ly_ctx_module_set_id; /**< module-set-id at the time ly_ctx_info was last cached */ + struct lyd_node *ly_ctx_info_cache; /**< a cache of calling ly_ctx_info on the ly_ctx */ +#endif pthread_rwlock_t ly_ctx_lock; /**< libyang's context rwlock */ }; extern struct np2srv np2srv; diff --git a/server/config.h.in b/server/config.h.in old mode 100644 new mode 100755 index fbb9f4202..e4b686a02 --- a/server/config.h.in +++ b/server/config.h.in @@ -52,4 +52,8 @@ */ #define NP2SRV_SR_LOCKED_RETRIES 3 +/** @brief Enable caching the ly_ctx_info() result + */ +#cmakedefine NP2SRV_ENABLED_LY_CTX_INFO_CACHE + #endif /* NP2SRV_CONFIG_H_ */ diff --git a/server/main.c b/server/main.c index ffd8f0268..6ed233ac2 100644 --- a/server/main.c +++ b/server/main.c @@ -434,14 +434,17 @@ np2srv_create_capab(const struct lys_module *mod) return cpb; } +#ifdef NP2SRV_ENABLED_LY_CTX_INFO_CACHE static void -np2srv_update_ly_ctx_info_cache() +np2srv_update_ly_ctx_info_cache(uint16_t module_set_id) { if (np2srv.ly_ctx_info_cache) { lyd_free_withsiblings(np2srv.ly_ctx_info_cache); } + np2srv.cached_ly_ctx_module_set_id = module_set_id; np2srv.ly_ctx_info_cache = ly_ctx_info(np2srv.ly_ctx); } +#endif static void np2srv_module_install_clb(const char *module_name, const char *revision, sr_module_state_t state, void *UNUSED(private_ctx)) @@ -525,7 +528,6 @@ np2srv_module_install_clb(const char *module_name, const char *revision, sr_modu free(cpb); } - np2srv_update_ly_ctx_info_cache(); /* unlock libyang context */ pthread_rwlock_unlock(&np2srv.ly_ctx_lock); @@ -560,7 +562,6 @@ np2srv_feature_change_clb(const char *module_name, const char *feature_name, boo lys_features_disable(mod, feature_name); } cpb = np2srv_create_capab(mod); - np2srv_update_ly_ctx_info_cache(); pthread_rwlock_unlock(&np2srv.ly_ctx_lock); np2srv_send_capab_change_notif(NULL, NULL, cpb); @@ -581,8 +582,17 @@ np2srv_state_data_clb(const char *xpath, sr_val_t **values, size_t *values_cnt, } else if (!strncmp(xpath, "/nc-notifications:", 18)) { data = ntf_get_data(); } else if (!strncmp(xpath, "/ietf-yang-library:", 19)) { +#ifdef NP2SRV_ENABLED_LY_CTX_INFO_CACHE + uint16_t module_set_id = ly_ctx_get_module_set_id(np2srv.ly_ctx); + if (module_set_id != np2srv.cached_ly_ctx_module_set_id) { + np2srv_update_ly_ctx_info_cache(module_set_id); + } + data = np2srv.ly_ctx_info_cache; should_free_data = false; +#else + data = ly_ctx_info(np2srv.ly_ctx); +#endif } else { ret = SR_ERR_OPERATION_FAILED; goto cleanup; @@ -1117,8 +1127,6 @@ np2srv_init_schemas(void) goto error; } - np2srv_update_ly_ctx_info_cache(); - /* debug - list schemas struct lyd_node *ylib = ly_ctx_info(np2srv.ly_ctx); lyd_print_file(stdout, ylib, LYD_JSON, LYP_WITHSIBLINGS); From e061725bd1211fa10c000afc37fa9b5f49ee0ed4 Mon Sep 17 00:00:00 2001 From: Michal Vasko Date: Tue, 14 Aug 2018 08:16:48 +0200 Subject: [PATCH 20/49] server VERSION bump to version 0.6.4 --- server/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/CMakeLists.txt b/server/CMakeLists.txt index ce4ffbca0..e2f2964cc 100755 --- a/server/CMakeLists.txt +++ b/server/CMakeLists.txt @@ -13,7 +13,7 @@ if(NOT UNIX) endif() # set version -set(NP2SRV_VERSION 0.6.3) +set(NP2SRV_VERSION 0.6.4) # set default build type if not specified by user if(NOT CMAKE_BUILD_TYPE) From b6d791099c1db6e33bb6f3a722e6ec8d14d9a439 Mon Sep 17 00:00:00 2001 From: Andrew Langefeld Date: Tue, 14 Aug 2018 17:33:44 -0500 Subject: [PATCH 21/49] Test to demonstrate container with default leaf This commit changes one of the test models such that it has a container with a leaf with a default. Prior to the cache optimizations in 6ab0617, an empty container whose leafs were all default values was suppressed and not generated as part of the running config. After 6ab0617, an empty container is now presented with the running-config. Thus `test_edit_delete1()` in `test_edit_get_config` will fail because the following empty container will be presented with the configuration: ``` ``` The tests pass in commits prior to 6ab0617. --- server/tests/files/test-feature-c.yin | 1 + 1 file changed, 1 insertion(+) diff --git a/server/tests/files/test-feature-c.yin b/server/tests/files/test-feature-c.yin index 929d76e7a..2af2c22d4 100755 --- a/server/tests/files/test-feature-c.yin +++ b/server/tests/files/test-feature-c.yin @@ -23,6 +23,7 @@ + From 0a8d2b74d4c679782a47765a3bf7a375536bbfb7 Mon Sep 17 00:00:00 2001 From: Michal Vasko Date: Wed, 15 Aug 2018 11:23:18 +0200 Subject: [PATCH 22/49] server BUGFIX reachable memory leak --- server/main.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/server/main.c b/server/main.c index 6ed233ac2..7ba02f049 100644 --- a/server/main.c +++ b/server/main.c @@ -1611,6 +1611,10 @@ main(int argc, char *argv[]) } nc_ps_free(np2srv.nc_ps); +#ifdef NP2SRV_ENABLED_LY_CTX_INFO_CACHE + lyd_free_withsiblings(np2srv.ly_ctx_info_cache); +#endif + /* clears all the sessions also */ sr_disconnect(np2srv.sr_conn); From 81fa32833503f8e820aaee400a44f0e926da86f9 Mon Sep 17 00:00:00 2001 From: Michal Vasko Date: Wed, 15 Aug 2018 11:42:23 +0200 Subject: [PATCH 23/49] server VERSION bump to version 0.6.5 --- server/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/CMakeLists.txt b/server/CMakeLists.txt index e2f2964cc..cd51e8846 100755 --- a/server/CMakeLists.txt +++ b/server/CMakeLists.txt @@ -13,7 +13,7 @@ if(NOT UNIX) endif() # set version -set(NP2SRV_VERSION 0.6.4) +set(NP2SRV_VERSION 0.6.5) # set default build type if not specified by user if(NOT CMAKE_BUILD_TYPE) From f8574754583e4b194a35a5401976ae3a3d221b43 Mon Sep 17 00:00:00 2001 From: Michal Vasko Date: Wed, 15 Aug 2018 15:36:04 +0200 Subject: [PATCH 24/49] server CHANGE handle new anydata value types --- server/op_copyconfig.c | 4 ++++ server/op_editconfig.c | 4 ++++ server/op_validate.c | 4 ++++ 3 files changed, 12 insertions(+) diff --git a/server/op_copyconfig.c b/server/op_copyconfig.c index 9a5b11c90..538c606fb 100644 --- a/server/op_copyconfig.c +++ b/server/op_copyconfig.c @@ -129,9 +129,13 @@ op_copyconfig(struct lyd_node *rpc, struct nc_session *ncs) case LYD_ANYDATA_XML: config = lyd_parse_xml(np2srv.ly_ctx, &any->value.xml, LYD_OPT_CONFIG | LYD_OPT_DESTRUCT | LYD_OPT_STRICT); break; + case LYD_ANYDATA_LYB: + config = lyd_parse_mem(np2srv.ly_ctx, any->value.mem, LYD_LYB, LYD_OPT_CONFIG | LYD_OPT_DESTRUCT | LYD_OPT_STRICT); + break; case LYD_ANYDATA_JSON: case LYD_ANYDATA_JSOND: case LYD_ANYDATA_SXMLD: + case LYD_ANYDATA_LYBD: EINT; ly_set_free(nodeset); e = nc_err(NC_ERR_OP_FAILED, NC_ERR_TYPE_APP); diff --git a/server/op_editconfig.c b/server/op_editconfig.c index 0cbfbe4ea..947e4a985 100644 --- a/server/op_editconfig.c +++ b/server/op_editconfig.c @@ -287,9 +287,13 @@ op_editconfig(struct lyd_node *rpc, struct nc_session *ncs) case LYD_ANYDATA_XML: config = lyd_parse_xml(np2srv.ly_ctx, &any->value.xml, LYD_OPT_EDIT | LYD_OPT_STRICT); break; + case LYD_ANYDATA_LYB: + config = lyd_parse_mem(np2srv.ly_ctx, any->value.mem, LYD_LYB, LYD_OPT_EDIT | LYD_OPT_STRICT); + break; case LYD_ANYDATA_JSON: case LYD_ANYDATA_JSOND: case LYD_ANYDATA_SXMLD: + case LYD_ANYDATA_LYBD: EINT; break; } diff --git a/server/op_validate.c b/server/op_validate.c index 1bb041811..5d10c3abc 100644 --- a/server/op_validate.c +++ b/server/op_validate.c @@ -71,9 +71,13 @@ op_validate(struct lyd_node *rpc, struct nc_session *ncs) case LYD_ANYDATA_XML: config = lyd_parse_xml(np2srv.ly_ctx, &any->value.xml, LYD_OPT_CONFIG | LYD_OPT_DESTRUCT | LYD_OPT_STRICT); break; + case LYD_ANYDATA_LYB: + config = lyd_parse_mem(np2srv.ly_ctx, any->value.mem, LYD_LYB, LYD_OPT_CONFIG | LYD_OPT_DESTRUCT | LYD_OPT_STRICT); + break; case LYD_ANYDATA_JSON: case LYD_ANYDATA_JSOND: case LYD_ANYDATA_SXMLD: + case LYD_ANYDATA_LYBD: EINT; e = nc_err(NC_ERR_OP_FAILED, NC_ERR_TYPE_APP); nc_err_set_msg(e, np2log_lasterr(np2srv.ly_ctx), "en"); From 186e6dcfb26ea2f03fb88cbbf82633c48ae2ff09 Mon Sep 17 00:00:00 2001 From: Michal Vasko Date: Wed, 15 Aug 2018 15:36:29 +0200 Subject: [PATCH 25/49] server VERSION bump to version 0.6.6 --- server/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/CMakeLists.txt b/server/CMakeLists.txt index cd51e8846..d5313a1a4 100755 --- a/server/CMakeLists.txt +++ b/server/CMakeLists.txt @@ -13,7 +13,7 @@ if(NOT UNIX) endif() # set version -set(NP2SRV_VERSION 0.6.5) +set(NP2SRV_VERSION 0.6.6) # set default build type if not specified by user if(NOT CMAKE_BUILD_TYPE) From a9844e220ca16f394262ea340aa3588de3a21f27 Mon Sep 17 00:00:00 2001 From: Michal Vasko Date: Wed, 15 Aug 2018 15:55:51 +0200 Subject: [PATCH 26/49] server BUGFIX keep dflt flags correctly set Fixes #316 --- server/operations.c | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/server/operations.c b/server/operations.c index 8426baec2..9317cb311 100644 --- a/server/operations.c +++ b/server/operations.c @@ -2822,7 +2822,7 @@ op_sr2ly(struct lyd_node *root, const sr_val_t *sr_val, struct lyd_node **new_no { char numstr[22]; char *val_str; - int new_node_i = -1; + int new_node_i = -1, parent_dflt; struct lyd_node *parent = NULL, *node; const struct lys_module *mod = NULL; @@ -2842,6 +2842,8 @@ op_sr2ly(struct lyd_node *root, const sr_val_t *sr_val, struct lyd_node **new_no /* get parent */ if (cache->used > 1) { parent = cache->items[cache->used - 2].node; + /* it will get erased when a child is inserted, set it back */ + parent_dflt = parent->dflt; } /* create the new node */ @@ -2885,6 +2887,10 @@ op_sr2ly(struct lyd_node *root, const sr_val_t *sr_val, struct lyd_node **new_no /* inherit dflt flag */ node->dflt = sr_val->dflt; + /* restore parent dflt flag */ + if (parent) { + parent->dflt = parent_dflt; + } /* insert into data tree if required */ if (!parent && root) { From b2aa69be8382a13dcab7ef7a02725c3e66cf94f5 Mon Sep 17 00:00:00 2001 From: Michal Vasko Date: Wed, 15 Aug 2018 15:56:39 +0200 Subject: [PATCH 27/49] server VERSION bump to version 0.6.7 --- server/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/CMakeLists.txt b/server/CMakeLists.txt index d5313a1a4..46b7b16fd 100755 --- a/server/CMakeLists.txt +++ b/server/CMakeLists.txt @@ -13,7 +13,7 @@ if(NOT UNIX) endif() # set version -set(NP2SRV_VERSION 0.6.6) +set(NP2SRV_VERSION 0.6.7) # set default build type if not specified by user if(NOT CMAKE_BUILD_TYPE) From 02cf25a1eb0e844f9b5b927f24e678cf11e2f14c Mon Sep 17 00:00:00 2001 From: Andrew Langefeld Date: Wed, 15 Aug 2018 18:11:10 -0500 Subject: [PATCH 28/49] Fix nested containers with default children Fixes #316. The previous fix in a9844e2 unfortunately only fixes the case where the container with the default child(ren) is at the top level of the tree. When there are containers with default child(ren) inside of other containers, all of the outer containers still do not have their default flag set correctly. This is because when `lyd_insert_common()` re-evaluates the default flag, it correctly updates not only the direct parent but also all more distant ancestors: ``` if (clrdflt) { /* remove the dflt flag from parents */ for (iter = parent; iter && iter->dflt; iter = iter->parent) { iter->dflt = 0; } } ``` This commit uses the same idea as a9844e2, but instead of saving only the immediate parent's `dflt` flag, it counts the number of non-default ancestors. After any node is inserted, it then re-walks the parents, reseting the default flag up to the correct depth. A better solution would probably be to modify `lyd_new_leaf()` and `lyd_new()` to take a `dflt` parameter, which could be passed from `op_sr2ly()`, and then _libyang_ would take care of keeping the default flags correct. But that would mean a _libyang_ public API change. This commit also modifies the test model again (compare #317) to have multiple nested children, and verifies the new fix. --- server/operations.c | 19 ++++++++++++------- server/tests/files/test-feature-c.yin | 14 ++++++++++++++ 2 files changed, 26 insertions(+), 7 deletions(-) diff --git a/server/operations.c b/server/operations.c index 9317cb311..44c43fd81 100644 --- a/server/operations.c +++ b/server/operations.c @@ -2822,8 +2822,8 @@ op_sr2ly(struct lyd_node *root, const sr_val_t *sr_val, struct lyd_node **new_no { char numstr[22]; char *val_str; - int new_node_i = -1, parent_dflt; - struct lyd_node *parent = NULL, *node; + int new_node_i = -1, parent_dflt_depth = 0; + struct lyd_node *parent = NULL, *node, *iter; const struct lys_module *mod = NULL; /* last used index in cache is ours, we can learn everything based on that */ @@ -2842,8 +2842,12 @@ op_sr2ly(struct lyd_node *root, const sr_val_t *sr_val, struct lyd_node **new_no /* get parent */ if (cache->used > 1) { parent = cache->items[cache->used - 2].node; - /* it will get erased when a child is inserted, set it back */ - parent_dflt = parent->dflt; + /* When children are inserted, the default flags will be cleared on all + parents, so determine how many parents' default flags we'll have + to reset. */ + for (iter = parent; iter && iter->dflt; iter = iter->parent) { + ++parent_dflt_depth; + } } /* create the new node */ @@ -2887,9 +2891,10 @@ op_sr2ly(struct lyd_node *root, const sr_val_t *sr_val, struct lyd_node **new_no /* inherit dflt flag */ node->dflt = sr_val->dflt; - /* restore parent dflt flag */ - if (parent) { - parent->dflt = parent_dflt; + + /* Restore the parent(s)' default flags as well */ + for (iter = parent; iter && parent_dflt_depth; iter = iter->parent, --parent_dflt_depth) { + iter->dflt = 1; } /* insert into data tree if required */ diff --git a/server/tests/files/test-feature-c.yin b/server/tests/files/test-feature-c.yin index 2af2c22d4..40ae2eaba 100755 --- a/server/tests/files/test-feature-c.yin +++ b/server/tests/files/test-feature-c.yin @@ -25,5 +25,19 @@ + + + + + + + + + + + + + + From 6e721210ef00882ff4fecf21899194b28869e382 Mon Sep 17 00:00:00 2001 From: Michal Vasko Date: Thu, 16 Aug 2018 08:14:02 +0200 Subject: [PATCH 29/49] server VERSION bump to version 0.6.8 --- server/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/CMakeLists.txt b/server/CMakeLists.txt index 46b7b16fd..f6cc0c414 100755 --- a/server/CMakeLists.txt +++ b/server/CMakeLists.txt @@ -13,7 +13,7 @@ if(NOT UNIX) endif() # set version -set(NP2SRV_VERSION 0.6.7) +set(NP2SRV_VERSION 0.6.8) # set default build type if not specified by user if(NOT CMAKE_BUILD_TYPE) From 429e60dcd439ee84bdec6e74345e4bbc0a5a7da7 Mon Sep 17 00:00:00 2001 From: Andrew Langefeld Date: Mon, 20 Aug 2018 11:45:54 -0500 Subject: [PATCH 30/49] Add subtree filter unit test This commit adds a more complex subtree filter test in `test_get`. The new test demonstrates a failure introduced in 6ab0617. When a get request is sent with the following filter: ``` complete ietf-yang-library ``` The following subtree should be returned: ``` complete ietf-yang-library 2018-01-17 ``` However, beginning in 6ab0617, the following malformed tree is returned: ``` complete ietf-yang-library 2018-01-17 ``` Notice the erroneous extra termination nodes. This test passes in commits prior to 6ab0617. It will fail beginning in 6ab0617. --- server/tests/test_get.c | 65 ++++++++++++++++++++++++++++++++++++++--- 1 file changed, 61 insertions(+), 4 deletions(-) mode change 100644 => 100755 server/tests/test_get.c diff --git a/server/tests/test_get.c b/server/tests/test_get.c old mode 100644 new mode 100755 index f2ba2d622..261e5d393 --- a/server/tests/test_get.c +++ b/server/tests/test_get.c @@ -149,10 +149,18 @@ __wrap_sr_get_item_next(sr_session_ctx_t *session, sr_val_iter_t *iter, sr_val_t char *path; (void)session; + /* Accept any queries in the ietf-yang-library namespace. */ if (!strncmp(xpath, "/ietf-yang-library:", 19)) { if (!set) { root = ly_ctx_info(np2srv.ly_ctx); - set = lyd_find_path(root, "//."); + /* Our test data only has information from the yang-library container, + so we can only service requests inside that container. But if the caller + has specified a more restrictive path inside yang-library, as in the case + of the filter tests, then use it. */ + if (strncmp(xpath, "/ietf-yang-library:yang-library", 31)) { + xpath = "/ietf-yang-library:yang-library//."; + } + set = lyd_find_path(root, xpath); } if (!set->number) { @@ -514,7 +522,7 @@ test_get(void **state) } static void -test_get_filter(void **state) +test_get_filter1(void **state) { (void)state; /* unused */ const char *get_rpc = @@ -533,12 +541,61 @@ test_get_filter(void **state) test_read(p_in, get_rpl, __LINE__); } +static void +test_get_filter2(void **state) +{ + (void)state; /* unused */ + const char *get_rpc = + "" + "" + "" + "" + "" + "complete" + "" + "ietf-yang-library" + "" + "" + "" + "" + "" + "" + ""; + const char *get_rpl = + "" + "" + "" + "" + "complete" + "" + "ietf-yang-library" + "2018-01-17" + "" + "" + "" + "" + ""; + + test_write(p_out, get_rpc, __LINE__); + test_read(p_in, get_rpl, __LINE__); +} + +static void +test_startstop(void **state) +{ + (void)state; /* unused */ + return; +} + int main(void) { const struct CMUnitTest tests[] = { - cmocka_unit_test_setup(test_get, np_start), - cmocka_unit_test_teardown(test_get_filter, np_stop), + cmocka_unit_test_setup(test_startstop, np_start), + cmocka_unit_test(test_get), + cmocka_unit_test(test_get_filter1), + cmocka_unit_test(test_get_filter2), + cmocka_unit_test_teardown(test_startstop, np_stop), }; if (setenv("CMOCKA_TEST_ABORT", "1", 1)) { From 5090741311581a922b3bedd43059ac3edf5524f2 Mon Sep 17 00:00:00 2001 From: Radek Krejci Date: Tue, 21 Aug 2018 10:36:59 +0200 Subject: [PATCH 31/49] cli BUGFIX dereferencing NULL pointer --- cli/commands.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cli/commands.c b/cli/commands.c index 58f0d4d74..cce922528 100644 --- a/cli/commands.c +++ b/cli/commands.c @@ -2402,7 +2402,7 @@ cmd_searchpath(const char *arg, char **UNUSED(tmp_config_file)) if (!arg[0]) { path = nc_client_get_schema_searchpath(); - fprintf(stdout, "%s\n", path[0] ? path : ""); + fprintf(stdout, "%s\n", path && path[0] ? path : ""); return 0; } From a554b0cd9b15a36e2975bfacf092342b03ce390b Mon Sep 17 00:00:00 2001 From: Radek Krejci Date: Tue, 21 Aug 2018 10:38:52 +0200 Subject: [PATCH 32/49] cli VERSION bump to version 2.0.49 --- cli/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cli/CMakeLists.txt b/cli/CMakeLists.txt index d6920ce0f..fcd9f1baf 100644 --- a/cli/CMakeLists.txt +++ b/cli/CMakeLists.txt @@ -21,7 +21,7 @@ set(CMAKE_C_FLAGS_RELEASE "-O2") set(CMAKE_C_FLAGS_DEBUG "-g -O0") # set version -set(NP2CLI_VERSION 2.0.48) +set(NP2CLI_VERSION 2.0.49) configure_file("${PROJECT_SOURCE_DIR}/version.h.in" "${PROJECT_BINARY_DIR}/version.h" ESCAPE_QUOTES @ONLY) include_directories(${PROJECT_BINARY_DIR}) From df9c784159e386d5859d4903dfc589f79520d18f Mon Sep 17 00:00:00 2001 From: Radek Krejci Date: Tue, 21 Aug 2018 10:39:24 +0200 Subject: [PATCH 33/49] server CHANGE ignore non-existing paths in 's filters Instead of returning error, just do not output any data for such a cases. --- server/op_get_config.c | 2 +- server/operations.c | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/server/op_get_config.c b/server/op_get_config.c index daa677bb0..dd80c88d8 100644 --- a/server/op_get_config.c +++ b/server/op_get_config.c @@ -51,7 +51,7 @@ opget_build_subtree_from_sysrepo(sr_session_ctx_t *srs, struct lyd_node **root, rc = np2srv_sr_get_items_iter(srs, full_subtree_xpath, &sriter, NULL); free(full_subtree_xpath); if (rc == 1) { - /* it's ok, model without data */ + /* it's ok, model without data or just non-existing path */ return 0; } else if (rc) { return -1; diff --git a/server/operations.c b/server/operations.c index 44c43fd81..f262fd002 100644 --- a/server/operations.c +++ b/server/operations.c @@ -479,7 +479,7 @@ np2srv_sr_get_change_next(sr_session_ctx_t *srs, sr_change_iter_t *iter, sr_chan return 0; } -/* UNKNOWN_MODEL, NOT_FOUND, and UNAUTHORIZED return 1, no error */ +/* UNKNOWN_MODEL, NOT_FOUND, UNKNOWN_MODEL, BAD_ELEMENT and UNAUTHORIZED return 1, no error */ int np2srv_sr_get_items_iter(sr_session_ctx_t *srs, const char *xpath, sr_val_iter_t **iter, struct nc_server_reply **ereply) { @@ -490,7 +490,8 @@ np2srv_sr_get_items_iter(sr_session_ctx_t *srs, const char *xpath, sr_val_iter_t if (!np2srv.disconnected) { rc = sr_get_items_iter(srs, xpath, iter); - if ((rc == SR_ERR_UNKNOWN_MODEL) || (rc == SR_ERR_NOT_FOUND) || (rc == SR_ERR_UNAUTHORIZED)) { + if ((rc == SR_ERR_UNKNOWN_MODEL) || (rc == SR_ERR_NOT_FOUND) || + (rc == SR_ERR_BAD_ELEMENT) || (rc == SR_ERR_UNKNOWN_MODEL) || (rc == SR_ERR_UNAUTHORIZED)) { pthread_rwlock_unlock(&sr_lock); return 1; } From 65da2b2c9228b4421fe5bb100d076408cb9b3669 Mon Sep 17 00:00:00 2001 From: Radek Krejci Date: Tue, 21 Aug 2018 10:41:29 +0200 Subject: [PATCH 34/49] server VERSION bump to version 0.6.9 --- server/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/CMakeLists.txt b/server/CMakeLists.txt index f6cc0c414..9b759cc76 100755 --- a/server/CMakeLists.txt +++ b/server/CMakeLists.txt @@ -13,7 +13,7 @@ if(NOT UNIX) endif() # set version -set(NP2SRV_VERSION 0.6.8) +set(NP2SRV_VERSION 0.6.9) # set default build type if not specified by user if(NOT CMAKE_BUILD_TYPE) From 426aba1bbd7dab4528296b1055e47167579fbb2c Mon Sep 17 00:00:00 2001 From: Michal Vasko Date: Tue, 21 Aug 2018 13:39:17 +0200 Subject: [PATCH 35/49] server BUGFIX use relative paths once parent exists Fixes #322 --- server/operations.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/server/operations.c b/server/operations.c index f262fd002..2cde6de4b 100644 --- a/server/operations.c +++ b/server/operations.c @@ -2809,6 +2809,9 @@ op_sr2ly_get_cache_parent(const char *xpath, struct sr2ly_cache *cache, struct l } ly_set_free(set); + /* keep track of the current cache parent (mainly so that we can distinguish relative/absolute paths) */ + ++i; + /* parse next node */ xpath = op_sr2ly_parse_node(xpath, &mod, &mod_len, &name, &name_len, &pred, &pred_len); } From 91cbf8f7858854a9e25de67a921ef245d2616a95 Mon Sep 17 00:00:00 2001 From: Michal Vasko Date: Tue, 21 Aug 2018 13:40:04 +0200 Subject: [PATCH 36/49] server VERSION bump to version 0.6.10 --- server/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/CMakeLists.txt b/server/CMakeLists.txt index 9b759cc76..d652789e3 100755 --- a/server/CMakeLists.txt +++ b/server/CMakeLists.txt @@ -13,7 +13,7 @@ if(NOT UNIX) endif() # set version -set(NP2SRV_VERSION 0.6.9) +set(NP2SRV_VERSION 0.6.10) # set default build type if not specified by user if(NOT CMAKE_BUILD_TYPE) From fea98b3a0473142a998d9578b3356fca9a15b8ed Mon Sep 17 00:00:00 2001 From: Andrew Langefeld Date: Mon, 27 Aug 2018 16:32:02 -0500 Subject: [PATCH 37/49] Provide the other certs from server's cert chain This commit from @jwwilcox, together with a corresponding commit to _libnetconf2_, fixes the TLS connection scenario in which the server's certificate has been signed by an intermediate CA, but the client only has the root CA available locally. In this case, the client will reject the connection attempt, because it does not know about the intermediate CA. This change adds an optional callback so that the server can supply the missing certificate. The changes in _libnetconf2_ will use the certificates retrieved from this callback to allow the server's TLS context to automatically provide the intermediate certificate(s) to the client. This scenario is demonstrated in the integration test `test_tls_client_missing_server_intermediate()` in [ADTRAN:netopeer2-integration-tests](https://github.com/ADTRAN/netopeer2-integration-tests/blob/master/tests/test_tls.py#L73). The changes here, together with the corresponding commit in _libnetconf2_, will allow [the currently failing test case](https://travis-ci.org/ADTRAN/netopeer2-integration-tests/jobs/420293391#L7434) to pass. --- server/ietf_keystore.c | 51 ++++++++++++++++++++++++++++++++++++ server/ietf_keystore.h | 3 +++ server/ietf_netconf_server.c | 1 + 3 files changed, 55 insertions(+) diff --git a/server/ietf_keystore.c b/server/ietf_keystore.c index ef11cf058..4d6960ba4 100644 --- a/server/ietf_keystore.c +++ b/server/ietf_keystore.c @@ -101,6 +101,57 @@ np_server_cert_clb(const char *name, void *UNUSED(user_data), char **UNUSED(cert return 0; } +int +np_server_cert_chain_clb(const char *name, void *UNUSED(user_data), char ***UNUSED(cert_paths), int *UNUSED(cert_path_count), + char ***cert_data, int *cert_data_count) +{ + int ret; + char *path; + sr_val_t *sr_certs; + size_t sr_cert_count, i, used_count; + + ret = asprintf(&path, "/ietf-keystore:keystore/private-keys/private-key/certificate-chains/" + "certificate-chain[name='%s']/certificate", name); + if (ret == -1) { + EMEM; + return 1; + } + + if (np2srv.sr_sess.ds != SR_DS_RUNNING) { + if (np2srv_sr_session_switch_ds(np2srv.sr_sess.srs, SR_DS_RUNNING, NULL)) { + free(path); + return 1; + } + np2srv.sr_sess.ds = SR_DS_RUNNING; + } + + /* Refresh the session to prevent sysrepo returning cached data */ + if (np2srv_sr_session_refresh(np2srv.sr_sess.srs, NULL)) { + ERR("%s:%d Failed session refresh", __func__, __LINE__); + free(path); + return 1; + } + + if (np2srv_sr_get_items(np2srv.sr_sess.srs, path, &sr_certs, &sr_cert_count, NULL)) { + free(path); + return 1; + } + free(path); + + /* Ignore the first cert since it's already loaded */ + if (sr_cert_count > 1) { + used_count = sr_cert_count - 1; + *cert_data = calloc(used_count, sizeof **cert_data); + for (i = 0; i < used_count; ++i) { + (*cert_data)[i] = strdup(sr_certs[i + 1].data.binary_val); + } + *cert_data_count = used_count; + } + + sr_free_values(sr_certs, sr_cert_count); + return 0; +} + int np_trusted_cert_list_clb(const char *name, void *UNUSED(user_data), char ***UNUSED(cert_paths), int *UNUSED(cert_path_count), char ***cert_data, int *cert_data_count) diff --git a/server/ietf_keystore.h b/server/ietf_keystore.h index 60a89e85a..2097776b9 100644 --- a/server/ietf_keystore.h +++ b/server/ietf_keystore.h @@ -20,6 +20,9 @@ int np_hostkey_clb(const char *name, void *user_data, char **privkey_path, char int np_server_cert_clb(const char *name, void *user_data, char **cert_path, char **cert_data, char **privkey_path, char **privkey_data, int *privkey_data_rsa); +int np_server_cert_chain_clb(const char *name, void *user_data, char ***cert_paths, int *cert_path_count, + char ***cert_data, int *cert_data_count); + int np_trusted_cert_list_clb(const char *name, void *user_data, char ***cert_paths, int *cert_path_count, char ***cert_data, int *cert_data_count); diff --git a/server/ietf_netconf_server.c b/server/ietf_netconf_server.c index 0f6e6cd12..37dc0c62f 100644 --- a/server/ietf_netconf_server.c +++ b/server/ietf_netconf_server.c @@ -1241,6 +1241,7 @@ ietf_netconf_server_init(const struct lys_module *module) nc_server_ssh_set_hostkey_clb(np_hostkey_clb, NULL, NULL); #ifdef NC_ENABLED_TLS nc_server_tls_set_server_cert_clb(np_server_cert_clb, NULL, NULL); + nc_server_tls_set_server_cert_chain_clb(np_server_cert_chain_clb, NULL, NULL); nc_server_tls_set_trusted_cert_list_clb(np_trusted_cert_list_clb, NULL, NULL); #endif From dda96fdceefe9c966be1fcd7a9ac648f6477babf Mon Sep 17 00:00:00 2001 From: Michal Vasko Date: Tue, 28 Aug 2018 12:22:38 +0200 Subject: [PATCH 38/49] server VERSION bump to version 0.6.11 --- server/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/CMakeLists.txt b/server/CMakeLists.txt index d652789e3..e436b876a 100755 --- a/server/CMakeLists.txt +++ b/server/CMakeLists.txt @@ -13,7 +13,7 @@ if(NOT UNIX) endif() # set version -set(NP2SRV_VERSION 0.6.10) +set(NP2SRV_VERSION 0.6.11) # set default build type if not specified by user if(NOT CMAKE_BUILD_TYPE) From e75e3391ca46faf0a02d73c985671ed04a408a85 Mon Sep 17 00:00:00 2001 From: Michal Vasko Date: Thu, 30 Aug 2018 10:29:20 +0200 Subject: [PATCH 39/49] server BUGFIX creating keys in case sysrepo will not return them Also a test case added that did not pass before. Fixes #329 --- server/operations.c | 120 +++++++++++++++++++++++++++++++++++++++- server/tests/test_get.c | 28 ++++++++++ 2 files changed, 147 insertions(+), 1 deletion(-) diff --git a/server/operations.c b/server/operations.c index 2cde6de4b..19b25dcfc 100644 --- a/server/operations.c +++ b/server/operations.c @@ -2631,6 +2631,85 @@ op_filter_create(struct lyd_node *filter_node, char ***filters, int *filter_coun return 0; } +static int +op_sr2ly_parse_pred(const char **pred, char **name, char **value) +{ + int len; + char quote; + + *name = NULL; + *value = NULL; + + if ((*pred)[0] != '[') { + goto error; + } + ++(*pred); + + for (len = 0; (*pred)[len] != '='; ++len); + + /* copy node name */ + *name = strndup(*pred, len); + + *pred += len; + + ++(*pred); + + if (((*pred)[0] != '\'') && ((*pred)[0] != '\"')) { + goto error; + } + + quote = (*pred)[0]; + ++(*pred); + + for (len = 0; (*pred)[len] != quote; ++len); + + /* copy value */ + *value = strndup(*pred, len); + + *pred += len; + + ++(*pred); + + if ((*pred)[0] != ']') { + goto error; + } + ++(*pred); + + if ((*pred)[0] != '[') { + /* no more predicates */ + *pred = NULL; + } + + return 0; + +error: + free(*name); + free(*value); + return -1; +} + +static int +op_sr2ly_create_keys(const char *pred, struct lyd_node *parent, const struct lys_module *module) +{ + char *name, *value; + struct lyd_node *node; + + while (pred) { + if (op_sr2ly_parse_pred(&pred, &name, &value)) { + return -1; + } + + node = lyd_new_leaf(parent, module, name, value); + free(name); + free(value); + if (!node) { + return -1; + } + } + + return 0; +} + static const char * op_sr2ly_parse_node(const char *xpath, const char **mod, int *mod_len, const char **name, int *name_len, const char **pred, int *pred_len) @@ -2803,6 +2882,12 @@ op_sr2ly_get_cache_parent(const char *xpath, struct sr2ly_cache *cache, struct l /* first created node */ *new_node_i = cache->used - 1; } + + /* create list keys if possible */ + if (op_sr2ly_create_keys(pred, cache->items[cache->used - 1].node, module)) { + ly_set_free(set); + return -1; + } } else { /* the parent exists, cache it */ cache->items[cache->used - 1].node = set->set.d[0]; @@ -2816,7 +2901,7 @@ op_sr2ly_get_cache_parent(const char *xpath, struct sr2ly_cache *cache, struct l xpath = op_sr2ly_parse_node(xpath, &mod, &mod_len, &name, &name_len, &pred, &pred_len); } - /* create item for the newly created node and return its index */ + /* create item for the newly created node */ op_sr2ly_add_cache(cache, mod, mod_len, name, name_len, pred, pred_len); return 0; } @@ -2826,15 +2911,40 @@ op_sr2ly(struct lyd_node *root, const sr_val_t *sr_val, struct lyd_node **new_no { char numstr[22]; char *val_str; + uint16_t i; int new_node_i = -1, parent_dflt_depth = 0; struct lyd_node *parent = NULL, *node, *iter; const struct lys_module *mod = NULL; + struct lys_node_list *list; /* last used index in cache is ours, we can learn everything based on that */ if (op_sr2ly_get_cache_parent(sr_val->xpath, cache, root, &new_node_i)) { return -1; } + /* if we wanted to create a list key, it exists already */ + if (!cache->items[cache->used - 1].mod && cache->items[cache->used - 2].pred) { + assert(cache->items[cache->used - 2].node->schema->nodetype == LYS_LIST); + + list = (struct lys_node_list *)cache->items[cache->used - 2].node->schema; + for (i = 0; i < list->keys_size; ++i) { + if (!strcmp(list->keys[i]->name, cache->items[cache->used - 1].name)) { + break; + } + } + + if (i < list->keys_size) { + if (new_node_i == -1) { + /* this is not the first created node, we can return whatever */ + *new_node = NULL; + } else { + /* return the list (or its parent) as we correctly should */ + *new_node = cache->items[new_node_i].node; + } + goto key_created; + } + } + /* get module */ if (cache->items[cache->used - 1].mod) { mod = ly_ctx_get_module(np2srv.ly_ctx, cache->items[cache->used - 1].mod, NULL, 1); @@ -2896,6 +3006,13 @@ op_sr2ly(struct lyd_node *root, const sr_val_t *sr_val, struct lyd_node **new_no /* inherit dflt flag */ node->dflt = sr_val->dflt; + if (sr_val->type == SR_LIST_T) { + /* create also all the keys of a list */ + if (op_sr2ly_create_keys(cache->items[cache->used - 1].pred, node, mod)) { + return -1; + } + } + /* Restore the parent(s)' default flags as well */ for (iter = parent; iter && parent_dflt_depth; iter = iter->parent, --parent_dflt_depth) { iter->dflt = 1; @@ -2920,6 +3037,7 @@ op_sr2ly(struct lyd_node *root, const sr_val_t *sr_val, struct lyd_node **new_no /* store in cache */ cache->items[cache->used - 1].node = node; } else { +key_created: /* useless to store ending nodes in cache */ free(cache->items[cache->used - 1].mod); free(cache->items[cache->used - 1].name); diff --git a/server/tests/test_get.c b/server/tests/test_get.c index 261e5d393..375272a34 100755 --- a/server/tests/test_get.c +++ b/server/tests/test_get.c @@ -580,6 +580,33 @@ test_get_filter2(void **state) test_read(p_in, get_rpl, __LINE__); } +static void +test_get_filter3(void **state) +{ + (void)state; /* unused */ + const char *get_rpc = + "" + "" + "" + "" + ""; + const char *get_rpl = + "" + "" + "" + "" + "complete" + "23" + "" + "" + "" + ""; + + test_write(p_out, get_rpc, __LINE__); + test_read(p_in, get_rpl, __LINE__); +} + static void test_startstop(void **state) { @@ -595,6 +622,7 @@ main(void) cmocka_unit_test(test_get), cmocka_unit_test(test_get_filter1), cmocka_unit_test(test_get_filter2), + cmocka_unit_test(test_get_filter3), cmocka_unit_test_teardown(test_startstop, np_stop), }; From 7da87c7d0975fec9d3d1d21d4365f99bacedf5b5 Mon Sep 17 00:00:00 2001 From: Michal Vasko Date: Thu, 30 Aug 2018 10:30:19 +0200 Subject: [PATCH 40/49] server VERSION bump to version 0.6.12 --- server/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/CMakeLists.txt b/server/CMakeLists.txt index e436b876a..fcfb671a9 100755 --- a/server/CMakeLists.txt +++ b/server/CMakeLists.txt @@ -13,7 +13,7 @@ if(NOT UNIX) endif() # set version -set(NP2SRV_VERSION 0.6.11) +set(NP2SRV_VERSION 0.6.12) # set default build type if not specified by user if(NOT CMAKE_BUILD_TYPE) From 206f985efddcec0fe883481947d840effd4719cc Mon Sep 17 00:00:00 2001 From: Andrew Langefeld Date: Thu, 30 Aug 2018 22:26:16 -0500 Subject: [PATCH 41/49] Fix multi-schema filtered get This commit fixes the use case for a get with a subtree filter containing nodes from more than one schema. This scenario worked correctly in older commits, but was broken by the optimizations in 6ab0617. The problem was that if there was a `root` tree already populated when `op_sr2ly()` was called, `op_sr2ly()` would correctly construct a new subtree with the requested data, but the subtree would never be attached to the existing `root` tree, and was essentially orphaned. Only the first tree connected to `root` would ever be returned. I fixed this by adding a call to `lyd_insert_after()` when a new top-level node was created but there was already data in `root`. This commit includes a test case that demonstrates the problem. The test case will pass in commits prior to 6ab0617, fail in 6ab0617, and then pass again with the code changes here. --- server/operations.c | 8 +++++ server/tests/test_edit_get_config.c | 56 +++++++++++++++++++++++++++-- 2 files changed, 61 insertions(+), 3 deletions(-) mode change 100644 => 100755 server/operations.c diff --git a/server/operations.c b/server/operations.c old mode 100644 new mode 100755 index 19b25dcfc..5ef182b4d --- a/server/operations.c +++ b/server/operations.c @@ -2873,6 +2873,14 @@ op_sr2ly_get_cache_parent(const char *xpath, struct sr2ly_cache *cache, struct l } else { /* top-level node */ cache->items[cache->used - 1].node = lyd_new(NULL, module, cache->items[cache->used - 1].name); + + /* If root already contains a data tree, insert this new top-level node */ + if (root && cache->items[cache->used - 1].node) { + if (lyd_insert_after(root->prev, cache->items[cache->used - 1].node)) { + ly_set_free(set); + return -1; + } + } } if (!cache->items[cache->used - 1].node) { ly_set_free(set); diff --git a/server/tests/test_edit_get_config.c b/server/tests/test_edit_get_config.c index c91925329..7d34877af 100755 --- a/server/tests/test_edit_get_config.c +++ b/server/tests/test_edit_get_config.c @@ -165,9 +165,15 @@ __wrap_sr_get_item_next(sr_session_ctx_t *session, sr_val_iter_t *iter, sr_val_t char *path; const char *ietf_interfaces_xpath = "/ietf-interfaces:"; size_t ietf_interfaces_xpath_len = strlen(ietf_interfaces_xpath); + const char *test_feature_c_xpath = "/test-feature-c:"; + size_t test_feature_c_xpath_len = strlen(test_feature_c_xpath); + const char *simplified_melt_xpath = "/simplified-melt:"; + size_t simplified_melt_xpath_len = strlen(simplified_melt_xpath); (void)session; - if (!strncmp(xpath, ietf_interfaces_xpath, ietf_interfaces_xpath_len) || !strcmp(xpath, "/test-feature-c:*//.") || !strcmp(xpath, "/simplified-melt:*//.")) { + if (!strncmp(xpath, ietf_interfaces_xpath, ietf_interfaces_xpath_len) || + !strncmp(xpath, test_feature_c_xpath, test_feature_c_xpath_len) || + !strncmp(xpath, simplified_melt_xpath, simplified_melt_xpath_len)) { if (!ietf_if_set) { ietf_if_set = lyd_find_path(data, xpath); } @@ -1546,7 +1552,7 @@ test_edit_merge3(void **state) } static void -test_get_filter(void **state) +test_get_filter1(void **state) { (void)state; /* unused */ const char *get_config_rpc = @@ -1582,6 +1588,49 @@ test_get_filter(void **state) test_read(p_in, get_config_rpl, __LINE__); } +static void +test_get_filter2(void **state) +{ + (void)state; /* unused */ + const char *get_config_rpc = + "" + "" + "" + "" + "" + "" + "" + "" + "" + "" + "" + "" + "" + "" + "" + "" + ""; + const char *get_config_rpl = + "" + "" + "" + "" + "iface2" + "" + "" + "iface1" + "" + "" + "" + "green" + "" + "" + ""; + + test_write(p_out, get_config_rpc, __LINE__); + test_read(p_in, get_config_rpl, __LINE__); +} + static void test_startstop(void **state) { @@ -1605,7 +1654,8 @@ main(void) cmocka_unit_test(test_edit_merge1), cmocka_unit_test(test_edit_merge2), cmocka_unit_test(test_edit_merge3), - cmocka_unit_test(test_get_filter), + cmocka_unit_test(test_get_filter1), + cmocka_unit_test(test_get_filter2), cmocka_unit_test_teardown(test_startstop, np_stop), }; From c1eb39546507dac034602274fccc3cf03ff08da4 Mon Sep 17 00:00:00 2001 From: Michal Vasko Date: Fri, 31 Aug 2018 09:36:03 +0200 Subject: [PATCH 42/49] server VERSION bump to version 0.6.13 --- server/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/CMakeLists.txt b/server/CMakeLists.txt index fcfb671a9..8efcd1b29 100755 --- a/server/CMakeLists.txt +++ b/server/CMakeLists.txt @@ -13,7 +13,7 @@ if(NOT UNIX) endif() # set version -set(NP2SRV_VERSION 0.6.12) +set(NP2SRV_VERSION 0.6.13) # set default build type if not specified by user if(NOT CMAKE_BUILD_TYPE) From 170a2ee6188db06a1bb228358ac4c116625eff87 Mon Sep 17 00:00:00 2001 From: Andrew Langefeld Date: Fri, 31 Aug 2018 17:51:09 -0500 Subject: [PATCH 43/49] Fix list without key This commit fixes the scenario in which the server needs to report a list without keys. (This is valid for lists that do not represent configuration; see RFC 7950 section 7.8.2.) This scenario was broken by the changes in e75e339, which caused `op_sr2ly()` to fail if the list's xpath predicate did not contain a `child="value"` expression. I made two modifications to `op_sr2ly_parse_pred()`: 1. Fix the loop that searches for `'='` so that it stops at the NULL termination of the string if no `'='` has yet been found. (The existing code could search far beyond the bounds of the string, sometimes resulting in a segmentation fault.) 2. Distinguish between a failure to find a `child="value"` predicate, which is a valid case, and other general parsing errors, which are not. I then changed `op_sr2ly_create_keys()` so that it no longer fails if no list keys are found. I also added a test case that demonstrates the problem. I simulated the `netconf-config-change` notification from RFC 6470, which contains a list without a key. The test case will pass in commits prior to e75e339, fail with e75e339, and pass again with this patch applied. --- server/operations.c | 26 +++++++++++--- server/tests/test_notif.c | 76 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 97 insertions(+), 5 deletions(-) mode change 100644 => 100755 server/tests/test_notif.c diff --git a/server/operations.c b/server/operations.c index 5ef182b4d..986d2c304 100755 --- a/server/operations.c +++ b/server/operations.c @@ -2631,11 +2631,18 @@ op_filter_create(struct lyd_node *filter_node, char ***filters, int *filter_coun return 0; } +enum { + OP_SR2LY_PARSE_PRED_SUCCESS = 0, + OP_SR2LY_PARSE_PRED_NO_KEY, + OP_SR2LY_PARSE_PRED_PARSE_ERROR +}; + static int op_sr2ly_parse_pred(const char **pred, char **name, char **value) { int len; char quote; + int ret = OP_SR2LY_PARSE_PRED_PARSE_ERROR; *name = NULL; *value = NULL; @@ -2645,7 +2652,12 @@ op_sr2ly_parse_pred(const char **pred, char **name, char **value) } ++(*pred); - for (len = 0; (*pred)[len] != '='; ++len); + for (len = 0; (*pred)[len] && (*pred)[len] != '='; ++len); + + if ((*pred)[len] != '=') { + ret = OP_SR2LY_PARSE_PRED_NO_KEY; + goto error; + } /* copy node name */ *name = strndup(*pred, len); @@ -2680,12 +2692,12 @@ op_sr2ly_parse_pred(const char **pred, char **name, char **value) *pred = NULL; } - return 0; + return OP_SR2LY_PARSE_PRED_SUCCESS; error: free(*name); free(*value); - return -1; + return ret; } static int @@ -2693,10 +2705,14 @@ op_sr2ly_create_keys(const char *pred, struct lyd_node *parent, const struct lys { char *name, *value; struct lyd_node *node; + int result; while (pred) { - if (op_sr2ly_parse_pred(&pred, &name, &value)) { - return -1; + result = op_sr2ly_parse_pred(&pred, &name, &value); + if (result != OP_SR2LY_PARSE_PRED_SUCCESS) { + /* It is legal for non-config lists to contain no keys, + so do not return a failure if no keys are found */ + return result == OP_SR2LY_PARSE_PRED_NO_KEY ? 0 : -1; } node = lyd_new_leaf(parent, module, name, value); diff --git a/server/tests/test_notif.c b/server/tests/test_notif.c old mode 100644 new mode 100755 index 990f114aa..178e4ae08 --- a/server/tests/test_notif.c +++ b/server/tests/test_notif.c @@ -370,6 +370,81 @@ test_basic(void **state) test_read(p_in, notif_data, __LINE__); } +static void +test_config_change(void **state) +{ + (void)state; /* unused */ + sr_val_t *vals; + size_t val_cnt; + const char *subsc_rpc = + "" + "" + "NETCONF" + "" + ""; + const char *subsc_rpl = + "" + "" + ""; + const char *notif_data = + "" + "0000-00-00T00:00:00Z" + "" + "" + "42" + "test" + "" + "running" + "" + "create" + "" + "" + ""; + + initialized = 0; + while (!initialized) { + usleep(100000); + } + + test_write(p_out, subsc_rpc, __LINE__); + test_read(p_in, subsc_rpl, __LINE__); + + /* send notif */ + val_cnt = 6; + vals = calloc(val_cnt, sizeof *vals); + + vals[0].xpath = strdup("/ietf-netconf-notifications:netconf-config-change/changed-by"); + vals[0].type = SR_CONTAINER_T; + + vals[1].xpath = strdup("/ietf-netconf-notifications:netconf-config-change/changed-by/session-id"); + vals[1].type = SR_UINT32_T; + vals[1].data.uint32_val = 42; + + vals[2].xpath = strdup("/ietf-netconf-notifications:netconf-config-change/changed-by/username"); + vals[2].type = SR_STRING_T; + vals[2].data.string_val = strdup("test"); + + vals[3].xpath = strdup("/ietf-netconf-notifications:netconf-config-change/datastore"); + vals[3].type = SR_ENUM_T; + vals[3].data.enum_val = strdup("running"); + + vals[4].xpath = strdup("/ietf-netconf-notifications:netconf-config-change/edit[1]"); + vals[4].type = SR_LIST_T; + + /* A real config-change notification will have an instance-id here, which is omitted for ease of testing. */ + + vals[5].xpath = strdup("/ietf-netconf-notifications:netconf-config-change/edit[1]/operation"); + vals[5].type = SR_ENUM_T; + vals[5].data.enum_val = strdup("create"); + + notif_clb(SR_EV_NOTIF_T_REALTIME, "/ietf-netconf-notifications:netconf-config-change", vals, val_cnt, time(NULL), notif_clb_data); + + sr_free_values(vals, val_cnt); + + /* read notif */ + test_read(p_in, notif_data, __LINE__); +} + static void test_filter_xpath(void **state) { @@ -630,6 +705,7 @@ main(void) { const struct CMUnitTest tests[] = { cmocka_unit_test_setup(test_basic, np_start), + cmocka_unit_test(test_config_change), cmocka_unit_test(test_filter_xpath), cmocka_unit_test(test_filter_subtree), cmocka_unit_test_teardown(test_replay, np_stop), From 3710ed8932e0a8d14e808126ab328a553caae240 Mon Sep 17 00:00:00 2001 From: Michal Vasko Date: Wed, 5 Sep 2018 09:10:20 +0200 Subject: [PATCH 44/49] server VERSION bump to version 0.6.14 --- server/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/CMakeLists.txt b/server/CMakeLists.txt index 8efcd1b29..38de513ac 100755 --- a/server/CMakeLists.txt +++ b/server/CMakeLists.txt @@ -13,7 +13,7 @@ if(NOT UNIX) endif() # set version -set(NP2SRV_VERSION 0.6.13) +set(NP2SRV_VERSION 0.6.14) # set default build type if not specified by user if(NOT CMAKE_BUILD_TYPE) From a3f63fa8026fadf7458b56624a513a05add39b56 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Kundr=C3=A1t?= Date: Thu, 6 Sep 2018 19:07:36 +0200 Subject: [PATCH 45/49] Adapt to libyang's "Pass the original user_data to the deleter function" --- server/main.c | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/server/main.c b/server/main.c index 7ba02f049..30d9b458b 100644 --- a/server/main.c +++ b/server/main.c @@ -336,13 +336,19 @@ np2srv_verify_clb(const struct nc_session *session) return 1; } +static void free_with_user_data(void *data, void *user_data) +{ + free(data); + (void)user_data; +} + static char * np2srv_ly_import_clb(const char *mod_name, const char *mod_rev, const char *submod_name, const char *submod_rev, - void *UNUSED(user_data), LYS_INFORMAT *format, void (**free_module_data)(void *model_data)) + void *UNUSED(user_data), LYS_INFORMAT *format, void (**free_module_data)(void *model_data, void *user_data)) { char *data = NULL; - *free_module_data = free; + *free_module_data = free_with_user_data; *format = LYS_YIN; if (submod_rev || (submod_name && !mod_name)) { np2srv_sr_get_submodule_schema(np2srv.sr_sess.srs, submod_name, submod_rev, SR_SCHEMA_YIN, &data, NULL); From 43b7ae6788d030b978a3d336d19fbbbb452be9e8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Kundr=C3=A1t?= Date: Thu, 6 Sep 2018 19:07:53 +0200 Subject: [PATCH 46/49] Adapt to libyang's "Change constness of the data returned by module implementation callbacks" --- server/main.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/main.c b/server/main.c index 30d9b458b..d344032f3 100644 --- a/server/main.c +++ b/server/main.c @@ -342,7 +342,7 @@ static void free_with_user_data(void *data, void *user_data) (void)user_data; } -static char * +static const char * np2srv_ly_import_clb(const char *mod_name, const char *mod_rev, const char *submod_name, const char *submod_rev, void *UNUSED(user_data), LYS_INFORMAT *format, void (**free_module_data)(void *model_data, void *user_data)) { From bf43336539210b4468097685819d5b901e60398c Mon Sep 17 00:00:00 2001 From: Michal Vasko Date: Fri, 7 Sep 2018 13:11:02 +0200 Subject: [PATCH 47/49] server VERSION bump to version 0.6.15 --- server/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/CMakeLists.txt b/server/CMakeLists.txt index 38de513ac..542bf05af 100755 --- a/server/CMakeLists.txt +++ b/server/CMakeLists.txt @@ -13,7 +13,7 @@ if(NOT UNIX) endif() # set version -set(NP2SRV_VERSION 0.6.14) +set(NP2SRV_VERSION 0.6.15) # set default build type if not specified by user if(NOT CMAKE_BUILD_TYPE) From 1ad19f2c1fc4ab6bc13777b0481865d797bf97ad Mon Sep 17 00:00:00 2001 From: Michal Vasko Date: Thu, 13 Sep 2018 09:00:33 +0200 Subject: [PATCH 48/49] keystored CHANGE generate new SSH hostkey, do not use the OpenSSH one Fixes #337 --- keystored/CMakeLists.txt | 7 +++++++ keystored/keystored.c | 6 +++--- keystored/scripts/ssh-key-import.sh | 14 ++++++++------ 3 files changed, 18 insertions(+), 9 deletions(-) diff --git a/keystored/CMakeLists.txt b/keystored/CMakeLists.txt index db4537af5..251dd9cc9 100644 --- a/keystored/CMakeLists.txt +++ b/keystored/CMakeLists.txt @@ -131,10 +131,17 @@ endif() option(SSH_KEY_INSTALL "Enable ssh key import" ON) if (SSH_KEY_INSTALL) + if (NOT SSH_KEYGEN_EXECUTABLE) + find_program(SSH_KEYGEN_EXECUTABLE ssh-keygen) + endif() + if (NOT SSH_KEYGEN_EXECUTABLE) + message(FATAL_ERROR "Unable to find ssh-keygen, set SSH_KEYGEN_EXECUTABLE manually.") + endif() install(CODE " set(ENV{SYSREPOCFG} ${SYSREPOCFG_EXECUTABLE}) set(ENV{CHMOD} ${CHMOD_EXECUTABLE}) set(ENV{OPENSSL} ${OPENSSL_EXECUTABLE}) + set(ENV{SSH_KEYGEN} ${SSH_KEYGEN_EXECUTABLE}) set(ENV{KEYSTORED_KEYS_DIR} ${KEYSTORED_KEYS_DIR}) set(ENV{KEYSTORED_CHECK_SSH_KEY} ${KEYSTORED_CHECK_SSH_KEY}) execute_process(COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/scripts/ssh-key-import.sh)") diff --git a/keystored/keystored.c b/keystored/keystored.c index 822174c81..f94813408 100644 --- a/keystored/keystored.c +++ b/keystored/keystored.c @@ -137,7 +137,7 @@ ks_privkey_get_cb(const char *xpath, sr_val_t **values, size_t *values_cnt, uint } name += 18; - if (asprintf(&path, "%s/%.*s.pub.pem", KEYSTORED_KEYS_DIR, (int)(strchr(name, '\'') - name), name) == -1) { + if (asprintf(&path, "%s/%.*s.pem.pub", KEYSTORED_KEYS_DIR, (int)(strchr(name, '\'') - name), name) == -1) { SRP_LOG_ERR("Memory allocation failed (%s:%d).", __FILE__, __LINE__); return SR_ERR_NOMEM; } @@ -337,7 +337,7 @@ ks_privkey_gen_cb(const char *UNUSED(xpath), const sr_node_t *input, const size_ goto cleanup; } sprintf(priv_path, "%s/%s.pem", KEYSTORED_KEYS_DIR, input[0].data.string_val); - sprintf(pub_path, "%s/%s.pub.pem", KEYSTORED_KEYS_DIR, input[0].data.string_val); + sprintf(pub_path, "%s/%s.pem.pub", KEYSTORED_KEYS_DIR, input[0].data.string_val); if (!(pid = fork())) { /* child */ @@ -451,7 +451,7 @@ ks_privkey_load_cb(const char *UNUSED(xpath), const sr_node_t *input, const size goto cleanup; } sprintf(priv_path, "%s/%s.pem", KEYSTORED_KEYS_DIR, input[0].data.string_val); - sprintf(pub_path, "%s/%s.pub.pem", KEYSTORED_KEYS_DIR, input[0].data.string_val); + sprintf(pub_path, "%s/%s.pem.pub", KEYSTORED_KEYS_DIR, input[0].data.string_val); fd = open(priv_path, O_CREAT | O_TRUNC | O_WRONLY, 00600); if (fd == -1) { diff --git a/keystored/scripts/ssh-key-import.sh b/keystored/scripts/ssh-key-import.sh index 8a3ba9aac..e3dd5320f 100755 --- a/keystored/scripts/ssh-key-import.sh +++ b/keystored/scripts/ssh-key-import.sh @@ -9,6 +9,7 @@ local_path=$(dirname $0) : ${SYSREPOCFG:=sysrepocfg} : ${CHMOD:=chmod} : ${OPENSSL:=openssl} +: ${SSH_KEYGEN:=ssh-keygen} : ${STOCK_KEY_CONFIG:=$local_path/../stock_key_config.xml} : ${KEYSTORED_KEYS_DIR:=/etc/keystored/keys} @@ -21,13 +22,14 @@ if [ $KEYSTORED_CHECK_SSH_KEY -eq 0 ]; then echo "- Warning: Assuming that an external script will provide the SSH key in a PEM format at \"${KEYSTORED_KEYS_DIR}/ssh_host_rsa_key.pem\"." echo "- Importing ietf-keystore stock key configuration..." $SYSREPOCFG -d startup -i ${STOCK_KEY_CONFIG} ietf-keystore -elif [ -r /etc/ssh/ssh_host_rsa_key ]; then - cp /etc/ssh/ssh_host_rsa_key ${KEYSTORED_KEYS_DIR}/ssh_host_rsa_key.pem +else + if [ -r ${KEYSTORED_KEYS_DIR}/ssh_host_rsa_key.pem -a -r ${KEYSTORED_KEYS_DIR}/ssh_host_rsa_key.pem.pub ]; then + echo "- SSH hostkey found, no need to generate a new one." + else + echo "- SSH hostkey not found, generating a new one..." + $SSH_KEYGEN -m pem -t rsa -q -N "" -f ${KEYSTORED_KEYS_DIR}/ssh_host_rsa_key.pem + fi $CHMOD go-rw ${KEYSTORED_KEYS_DIR}/ssh_host_rsa_key.pem - $OPENSSL rsa -pubout -in ${KEYSTORED_KEYS_DIR}/ssh_host_rsa_key.pem \ - -out ${KEYSTORED_KEYS_DIR}/ssh_host_rsa_key.pub.pem echo "- Importing ietf-keystore stock key configuration..." $SYSREPOCFG -d startup -i ${STOCK_KEY_CONFIG} ietf-keystore -else - echo "- Warning: Cannot read the SSH hostkey at /etc/ssh/ssh_host_rsa_key, skipping." fi From ca0256d4003c61781116e2a916e0c72a9e35ef8e Mon Sep 17 00:00:00 2001 From: Michal Vasko Date: Thu, 13 Sep 2018 09:01:25 +0200 Subject: [PATCH 49/49] keystored VERSION bump to version 0.1.2 --- keystored/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/keystored/CMakeLists.txt b/keystored/CMakeLists.txt index 251dd9cc9..33aec1e23 100644 --- a/keystored/CMakeLists.txt +++ b/keystored/CMakeLists.txt @@ -21,7 +21,7 @@ set(CMAKE_C_FLAGS_RELEASE "-O2 -DNDEBUG") set(CMAKE_C_FLAGS_DEBUG "-g -O0 -DDEBUG") # set version -set(KEYSTORED_VERSION 0.1.1) +set(KEYSTORED_VERSION 0.1.2) # config variables if (NOT KEYSTORED_KEYS_DIR)