Merge pull request #20 from snipsco/release/0.2.2

Release/0.2.2

CHANGELOG.md

@@ -1,6 +1,14 @@
 # Changelog
 All notable changes to this project will be documented in this file.
 
+## [0.2.2] - 2019-06-18
+### Added
+- Gazetteer entity extension API [#18](https://github.com/snipsco/snips-nlu-parsers/pull/18))
+
+### Changed
+- Bumped Rustling to `0.18.1` [#20](https://github.com/snipsco/snips-nlu-parsers/pull/20)
+- Extended Python dependencies version upper bound [#19](https://github.com/snipsco/snips-nlu-parsers/pull/19)
+
 ## [0.2.1] - 2019-04-08
 ### Added
 - Expose complete and by language builtin entities json configuration retrieval API [#13](https://github.com/snipsco/snips-nlu-parsers/pull/13)
@@ -23,6 +31,7 @@ All notable changes to this project will be documented in this file.
 - bump `snips-nlu-ontology` to `0.63.0`
 - re-export `gazetteer-entity-parser` crate
 
+[0.2.2]: https://github.com/snipsco/snips-nlu-parsers/compare/0.2.1...0.2.2
 [0.2.1]: https://github.com/snipsco/snips-nlu-parsers/compare/0.2.0...0.2.1
 [0.2.0]: https://github.com/snipsco/snips-nlu-parsers/compare/0.1.2...0.2.0
 [0.1.2]: https://github.com/snipsco/snips-nlu-parsers/compare/0.1.1...0.1.2

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "snips-nlu-parsers"
-version = "0.2.1"
+version = "0.2.2"
 authors = ["Adrien Ball <[email protected]>"]
 edition = "2018"
 
@@ -20,9 +20,9 @@ regex = "0.2"
 serde = "1.0"
 serde_derive = "1.0"
 serde_json = "1.0"
-gazetteer-entity-parser = { git = "https://github.com/snipsco/gazetteer-entity-parser", tag = "0.6.0" }
-rustling-ontology = { git = "https://github.com/snipsco/rustling-ontology", tag = "0.18.0" }
-snips-nlu-ontology = { git = "https://github.com/snipsco/snips-nlu-ontology", tag = "0.64.6" }
+gazetteer-entity-parser = { git = "https://github.com/snipsco/gazetteer-entity-parser", tag = "0.7.0" }
+rustling-ontology = { git = "https://github.com/snipsco/rustling-ontology", tag = "0.18.1" }
+snips-nlu-ontology = { git = "https://github.com/snipsco/snips-nlu-ontology", tag = "0.64.7" }
 snips-nlu-utils = { git = "https://github.com/snipsco/snips-nlu-utils", tag = "0.8.0" }
 
 [dev-dependencies]

ffi/Cargo.toml

@@ -1,15 +1,15 @@
 [package]
 name = "snips-nlu-parsers-ffi"
-version = "0.2.1"
+version = "0.2.2"
 authors = ["Adrien Ball <[email protected]>"]
 edition = "2018"
 
 [dependencies]
 failure = "0.1"
 ffi-utils = { git = "https://github.com/snipsco/snips-utils-rs", rev = "291ce1d" }
 libc = "0.2"
-snips-nlu-ontology = { git = "https://github.com/snipsco/snips-nlu-ontology", tag = "0.64.6" }
-snips-nlu-ontology-ffi-macros = { git = "https://github.com/snipsco/snips-nlu-ontology", tag = "0.64.6" }
+snips-nlu-ontology = { git = "https://github.com/snipsco/snips-nlu-ontology", tag = "0.64.7" }
+snips-nlu-ontology-ffi-macros = { git = "https://github.com/snipsco/snips-nlu-ontology", tag = "0.64.7" }
 snips-nlu-parsers-ffi-macros = { path = "ffi-macros" }
 
 [lib]

ffi/ffi-macros/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "snips-nlu-parsers-ffi-macros"
-version = "0.2.1"
+version = "0.2.2"
 authors = ["Adrien Ball <[email protected]>"]
 edition = "2018"
 
@@ -10,8 +10,8 @@ ffi-utils = { git = "https://github.com/snipsco/snips-utils-rs", rev = "291ce1d"
 libc = "0.2"
 serde = "1.0"
 serde_json = "1.0"
-snips-nlu-ontology = { git = "https://github.com/snipsco/snips-nlu-ontology", tag = "0.64.6" }
-snips-nlu-ontology-ffi-macros = { git = "https://github.com/snipsco/snips-nlu-ontology", tag = "0.64.6" }
+snips-nlu-ontology = { git = "https://github.com/snipsco/snips-nlu-ontology", tag = "0.64.7" }
+snips-nlu-ontology-ffi-macros = { git = "https://github.com/snipsco/snips-nlu-ontology", tag = "0.64.7" }
 snips-nlu-parsers = { path = "../.." }
 
 [lib]

ffi/ffi-macros/src/builtin_entity_parser.rs

@@ -2,14 +2,15 @@ use std::ffi::CStr;
 use std::slice;
 
 use failure::ResultExt;
+use ffi_utils::{convert_to_c_string, CReprOf, CStringArray, RawPointerConverter};
 use libc;
 use serde_json;
+use snips_nlu_ontology::{BuiltinEntity, BuiltinEntityKind, BuiltinGazetteerEntityKind};
+use snips_nlu_ontology_ffi_macros::{CBuiltinEntity, CBuiltinEntityArray};
+
+use snips_nlu_parsers::{BuiltinEntityParser, BuiltinEntityParserLoader, EntityValue};
 
 use crate::Result;
-use ffi_utils::{convert_to_c_string, CReprOf, CStringArray, RawPointerConverter};
-use snips_nlu_ontology::{BuiltinEntity, BuiltinEntityKind};
-use snips_nlu_ontology_ffi_macros::{CBuiltinEntity, CBuiltinEntityArray};
-use snips_nlu_parsers::{BuiltinEntityParser, BuiltinEntityParserLoader};
 
 #[repr(C)]
 pub struct CBuiltinEntityParser(*const libc::c_void);
@@ -22,6 +23,14 @@ macro_rules! get_parser {
     }};
 }
 
+macro_rules! get_parser_mut {
+    ($opaque:ident) => {{
+        let container: &$crate::CBuiltinEntityParser = unsafe { &*$opaque };
+        let x = container.0 as *mut BuiltinEntityParser;
+        unsafe { &mut *x }
+    }};
+}
+
 pub fn create_builtin_entity_parser(
     ptr: *mut *const CBuiltinEntityParser,
     json_config: *const libc::c_char,
@@ -38,6 +47,21 @@ pub fn create_builtin_entity_parser(
     Ok(())
 }
 
+pub fn extend_gazetteer_entity_json(
+    ptr: *const CBuiltinEntityParser,
+    entity_name: *const libc::c_char,
+    entity_values_json: *const libc::c_char,
+) -> Result<()> {
+    let parser = get_parser_mut!(ptr);
+    let entity_identifier = unsafe { CStr::from_ptr(entity_name) }.to_str()?;
+    let entity_kind = BuiltinGazetteerEntityKind::from_identifier(entity_identifier)?;
+    let entity_values_json_str = unsafe { CStr::from_ptr(entity_values_json) }.to_str()?;
+    let entity_values: Vec<EntityValue> = serde_json::from_str(entity_values_json_str)?;
+
+    parser.extend_gazetteer_entity(entity_kind, entity_values.into_iter())?;
+    Ok(())
+}
+
 pub fn persist_builtin_entity_parser(
     ptr: *const CBuiltinEntityParser,
     path: *const libc::c_char,

ffi/ffi-macros/src/gazetteer_entity_parser.rs

@@ -1,13 +1,14 @@
 use std::ffi::CStr;
 use std::slice;
 
+use ffi_utils::{convert_to_c_string, CReprOf, CStringArray, RawPointerConverter};
 use libc;
 use serde_json;
 
-use crate::Result;
-use ffi_utils::{convert_to_c_string, CReprOf, CStringArray, RawPointerConverter};
 use snips_nlu_parsers::{GazetteerEntityMatch, GazetteerParser, GazetteerParserBuilder};
 
+use crate::Result;
+
 #[repr(C)]
 pub struct CGazetteerEntityParser(*const libc::c_void);
 

ffi/ffi-macros/src/lib.rs

@@ -1,9 +1,9 @@
-mod builtin_entity_parser;
-mod gazetteer_entity_parser;
-
 pub use builtin_entity_parser::*;
 pub use gazetteer_entity_parser::*;
 
+mod builtin_entity_parser;
+mod gazetteer_entity_parser;
+
 type Result<T> = ::std::result::Result<T, ::failure::Error>;
 
 #[macro_export]
@@ -33,6 +33,19 @@ macro_rules! export_nlu_parsers_c_symbols {
             wrap!($crate::load_builtin_entity_parser(ptr, parser_path))
         }
 
+        #[no_mangle]
+        pub extern "C" fn snips_nlu_parsers_extend_gazetteer_entity_json(
+            ptr: *const $crate::CBuiltinEntityParser,
+            entity_name: *const libc::c_char,
+            entity_values_json: *const libc::c_char,
+        ) -> ::ffi_utils::SNIPS_RESULT {
+            wrap!($crate::extend_gazetteer_entity_json(
+                ptr,
+                entity_name,
+                entity_values_json
+            ))
+        }
+
         #[no_mangle]
         pub extern "C" fn snips_nlu_parsers_extract_builtin_entities(
             ptr: *const $crate::CBuiltinEntityParser,

ffi/src/lib.rs

@@ -1,7 +1,8 @@
 use ffi_utils::{generate_error_handling, wrap};
-use snips_nlu_parsers_ffi_macros::export_nlu_parsers_c_symbols;
 use snips_nlu_ontology_ffi_macros::export_nlu_ontology_c_symbols;
 
+use snips_nlu_parsers_ffi_macros::export_nlu_parsers_c_symbols;
+
 generate_error_handling!(snips_nlu_parsers_get_last_error);
 
 export_nlu_ontology_c_symbols!();

python/ffi/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "snips-nlu-parsers-python-ffi"
-version = "0.2.1"
+version = "0.2.2"
 authors = ["Adrien Ball <[email protected]>"]
 edition = "2018"
 
@@ -12,6 +12,6 @@ crate-type = ["cdylib"]
 failure = "0.1"
 libc = "0.2"
 ffi-utils = { git = "https://github.com/snipsco/snips-utils-rs", rev = "291ce1d" }
-snips-nlu-parsers-ffi-macros = { git = "https://github.com/snipsco/snips-nlu-parsers", tag = "0.2.1" }
-snips-nlu-ontology = { git = "https://github.com/snipsco/snips-nlu-ontology", tag = "0.64.6" }
-snips-nlu-ontology-ffi-macros = { git = "https://github.com/snipsco/snips-nlu-ontology", tag = "0.64.6" }
+snips-nlu-parsers-ffi-macros = { git = "https://github.com/snipsco/snips-nlu-parsers", tag = "0.2.2" }
+snips-nlu-ontology = { git = "https://github.com/snipsco/snips-nlu-ontology", tag = "0.64.7" }
+snips-nlu-ontology-ffi-macros = { git = "https://github.com/snipsco/snips-nlu-ontology", tag = "0.64.7" }

python/ffi/src/lib.rs

@@ -1,5 +1,6 @@
 use ffi_utils::*;
 use snips_nlu_ontology_ffi_macros::export_nlu_ontology_c_symbols;
+
 use snips_nlu_parsers_ffi_macros::export_nlu_parsers_c_symbols;
 
 generate_error_handling!(snips_nlu_parsers_get_last_error);

python/setup.py

@@ -28,8 +28,8 @@
     readme = f.read()
 
 required = [
-    "future==0.16.0",
-    "pathlib==1.0.1; python_version < '3.4'",
+    "future>=0.16,<0.18",
+    "pathlib>=1.0,<2.0; python_version < '3.4'",
 ]
 
 rust_extension = RustExtension(

python/snips_nlu_parsers/__version__

@@ -1 +1 @@
-0.2.1
+0.2.2

python/snips_nlu_parsers/builtin_entity_parser.py

@@ -1,5 +1,5 @@
 import json
-from _ctypes import byref, pointer
+from _ctypes import byref
 from builtins import bytes, str
 from ctypes import c_char_p, c_int, c_void_p, string_at
 from pathlib import Path
@@ -14,7 +14,7 @@ def __init__(self, parser):
 
     @classmethod
     def build(cls, language, gazetteer_entity_parser_path=None):
-        """Build a `BuiltinEntityParser`
+        """Builds a `BuiltinEntityParser`
 
         Args:
             language (str): Language identifier
@@ -30,7 +30,7 @@ def build(cls, language, gazetteer_entity_parser_path=None):
         parser_config = dict(
             language=language.upper(),
             gazetteer_parser_path=gazetteer_entity_parser_path)
-        parser = pointer(c_void_p())
+        parser = c_void_p()
         json_parser_config = bytes(json.dumps(parser_config), encoding="utf8")
         exit_code = lib.snips_nlu_parsers_create_builtin_entity_parser(
             byref(parser), json_parser_config)
@@ -39,7 +39,7 @@ def build(cls, language, gazetteer_entity_parser_path=None):
         return cls(parser)
 
     def parse(self, text, scope=None):
-        """Extract builtin entities from *text*
+        """Extracts builtin entities from *text*
 
         Args:
             text (str): Input
@@ -72,8 +72,32 @@ def parse(self, text, scope=None):
             result = string_at(ptr)
             return json.loads(result.decode("utf8"))
 
+    def extend_gazetteer_entity(self, entity_name, entity_values):
+        """Extends a builtin gazetteer entity with custom values
+
+        Args:
+            entity_name (str): Gazetteer entity identifier
+            entity_values (list of dict): List of entity values represented as
+                dictionaries with a 'raw_value' key and a 'resolved_value' key
+
+        Returns:
+            The same object, updated.
+
+        Raises:
+            ValueError: when the entity name is unknown or not present in the
+                parser
+        """
+        if not entity_values:
+            return self
+        entity_values_json = bytes(json.dumps(entity_values), encoding="utf8")
+        exit_code = lib.snips_nlu_parsers_extend_gazetteer_entity_json(
+            self._parser, entity_name.encode("utf8"), entity_values_json)
+        check_ffi_error(exit_code, "Something went wrong when extending the "
+                                   "builtin entity '%s'" % entity_name)
+        return self
+
     def persist(self, path):
-        """Persist the gazetteer parser on disk at the provided path"""
+        """Persists the builtin entity parser on disk at the provided path"""
         if isinstance(path, Path):
             path = str(path)
         exit_code = lib.snips_nlu_parsers_persist_builtin_entity_parser(
@@ -83,12 +107,12 @@ def persist(self, path):
 
     @classmethod
     def from_path(cls, parser_path):
-        """Create a :class:`GazetteerEntityParser` from a gazetteer parser
+        """Creates a :class:`BuiltinEntityParser` from a builtin entity parser
         persisted on disk
         """
         if isinstance(parser_path, Path):
             parser_path = str(parser_path)
-        parser = pointer(c_void_p())
+        parser = c_void_p()
         parser_path = bytes(parser_path, encoding="utf8")
         exit_code = lib.snips_nlu_parsers_load_builtin_entity_parser(
             byref(parser), parser_path)
@@ -97,5 +121,5 @@ def from_path(cls, parser_path):
         return cls(parser)
 
     def __del__(self):
-        if lib is not None and hasattr(self, '_parser'):
+        if lib is not None and self._parser is not None:
             lib.snips_nlu_parsers_destroy_builtin_entity_parser(self._parser)

python/snips_nlu_parsers/gazetteer_entity_parser.py

@@ -1,11 +1,11 @@
 import json
-from _ctypes import byref, pointer
+from _ctypes import byref
 from builtins import bytes, str
 from ctypes import c_char_p, c_int, c_void_p, string_at
 from pathlib import Path
 
 from snips_nlu_parsers.utils import (CStringArray, check_ffi_error, lib,
-                                    string_pointer)
+                                     string_pointer)
 
 
 class GazetteerEntityParser(object):
@@ -55,7 +55,7 @@ def build(cls, build_config):
                 ]
             }
         """
-        parser = pointer(c_void_p())
+        parser = c_void_p()
         json_parser_config = bytes(json.dumps(build_config), encoding="utf8")
         exit_code = lib.snips_nlu_parsers_build_gazetteer_entity_parser(
             byref(parser), json_parser_config)
@@ -113,7 +113,7 @@ def from_path(cls, parser_path):
         """
         if isinstance(parser_path, Path):
             parser_path = str(parser_path)
-        parser = pointer(c_void_p())
+        parser = c_void_p()
         parser_path = bytes(parser_path, encoding="utf8")
         exit_code = lib.snips_nlu_parsers_load_gazetteer_entity_parser(
             byref(parser), parser_path)