WIP: Migrate towards ljsonv3

landmarkerio/__init__.py

@@ -17,14 +17,15 @@ class Server(object):
     allowed_origins = ['https://www.landmarker.io',      # secure client
                        'http://localhost:4000',          # client development
                        'http://insecure.landmarker.io']  # legacy client
-    endpoint = '/api/v2/'
+    endpoint = '/api/v3/'
 
 
 class Endpoints(object):
     mode = 'mode'
     images = 'images'
     collections = 'collections'
     landmarks = 'landmarks'
+    landmarks_json = 'landmarks/json'
     meshes = 'meshes'
     templates = 'templates'
     points = 'points'

landmarkerio/landmark.py

@@ -30,11 +30,11 @@ def lm_ids(self, asset_id):
         pass
 
     @abc.abstractmethod
-    def load_lm(self, asset_id, lm_id):
+    def load_lm(self, asset_id):
         pass
 
     @abc.abstractmethod
-    def save_lm(self, asset_id, lm_id, lm_json):
+    def save_lm(self, asset_id, lm_json):
         pass
 
 
@@ -44,25 +44,25 @@ class FileLmAdapter(LmAdapter):
     local filesystem.
     """
 
-    def load_lm(self, asset_id, lm_id):
-        fp = self.lm_fp(asset_id, lm_id)
+    def load_lm(self, asset_id):
+        fp = self.lm_fp(asset_id)
         if not p.isfile(fp):
             raise IOError
         with open(fp, 'rb') as f:
             lm = json.load(f)
             return lm
 
-    def save_lm(self, asset_id, lm_id, lm_json):
+    def save_lm(self, asset_id, lm_json):
         r"""
         Persist a given landmark definition to disk.
         """
-        fp = self.lm_fp(asset_id, lm_id)
+        fp = self.lm_fp(asset_id)
         with open(fp, 'wb') as f:
             json.dump(lm_json, f, sort_keys=True, indent=4,
                       separators=(',', ': '))
 
     @abc.abstractmethod
-    def lm_fp(self, asset_id, lm_id):
+    def lm_fp(self, asset_id):
         # where a landmark should exist
         pass
 
@@ -79,9 +79,9 @@ def __init__(self, lm_dir):
             os.mkdir(self.lm_dir)
         print('landmarks: {}'.format(self.lm_dir))
 
-    def lm_fp(self, asset_id, lm_id):
+    def lm_fp(self, asset_id):
         # where a landmark should exist
-        return safe_join(safe_join(self.lm_dir, asset_id), lm_id + FileExt.lm)
+        return safe_join(safe_join(self.lm_dir, asset_id), FileExt.lm)
 
     def lm_ids(self, asset_id):
         r"""
@@ -112,14 +112,14 @@ def _lm_paths(self, asset_id=None):
         return filter(lambda f: p.isfile(f) and
                                 p.splitext(f)[-1] == FileExt.lm, g)
 
-    def save_lm(self, asset_id, lm_id, lm_json):
+    def save_lm(self, asset_id, lm_json):
         r"""
         Persist a given landmark definition to disk.
         """
         subject_dir = safe_join(self.lm_dir, asset_id)
         if not p.isdir(subject_dir):
             os.mkdir(subject_dir)
-        super(SeparateDirFileLmAdapter, self).save_lm(asset_id, lm_id, lm_json)
+        super(SeparateDirFileLmAdapter, self).save_lm(asset_id, lm_json)
 
 
 class InplaceFileLmAdapter(FileLmAdapter):
@@ -145,8 +145,7 @@ def asset_id_to_lm_id(self):
                 for aid in self.ids_to_paths
                 if self._lm_path_for_asset_id(aid).is_file()}
 
-    def lm_fp(self, asset_id, lm_id):
-        # note the lm_id is ignored. We just always return the .ljson file.
+    def lm_fp(self, asset_id):
         return str(self._lm_path_for_asset_id(asset_id))
 
     def _lm_path_for_asset_id(self, asset_id):

landmarkerio/server.py

@@ -154,26 +154,28 @@ def add_lm_endpoints(api, lm_adapter, template_adapter):
 
     class Landmark(Resource):
 
-        def get(self, asset_id, lm_id):
-            err = "{} does not have {} landmarks".format(asset_id, lm_id)
+        def get(self, asset_id):
+            err = "{} does not have landmarks".format(asset_id)
             try:
-                return lm_adapter.load_lm(asset_id, lm_id)
+                lm_json = lm_adapter.load_lm(asset_id)
+                template_adapter.repair_lm(lm_json)
+                return lm_json
             except Exception as e:
                 try:
-                    return template_adapter.load_template(lm_id)
+                    return template_adapter.load_templates()
                 except Exception as e:
                     return abort(404, message=err)
 
-        def put(self, asset_id, lm_id):
+        def put(self, asset_id):
             try:
-                return lm_adapter.save_lm(asset_id, lm_id, request.json)
+                return lm_adapter.save_lm(asset_id, request.json)
             except Exception as e:
                 print(e)
-                return abort(409, message="{}:{} unable to "
-                                          "save".format(asset_id, lm_id))
+                return abort(409, message="{} unable to "
+                                          "save".format(asset_id))
 
         # Need this here to enable CORS put see http://mzl.la/1rCDkWX
-        def options(self, asset_id, lm_id):
+        def options(self, asset_id):
             pass
 
     class LandmarkList(Resource):
@@ -187,27 +189,21 @@ def get(self, asset_id):
             return lm_adapter.lm_ids(asset_id)
 
     lm_url = partial(url, Endpoints.landmarks)
+    lm_json_url = partial(url, Endpoints.landmarks_json)
     api.add_resource(LandmarkList, lm_url())
     api.add_resource(LandmarkListForId, asset(lm_url)())
-    api.add_resource(Landmark, asset(lm_url)('<string:lm_id>'))
+    api.add_resource(Landmark, asset(lm_json_url)())
 
 
 def add_template_endpoints(api, adapter):
 
-    class Template(Resource):
-
-        def get(self, lm_id):
-            err = "{} template does not exist".format(lm_id)
-            return safe_send(adapter.load_template(lm_id), err)
-
     class TemplateList(Resource):
 
         def get(self):
             return adapter.template_ids()
 
     templates_url = partial(url, Endpoints.templates)
     api.add_resource(TemplateList, templates_url())
-    api.add_resource(Template, templates_url('<string:lm_id>'))
 
 
 def add_collection_endpoints(api, adapter):

landmarkerio/template.py

@@ -31,37 +31,41 @@ def parse_connectivity(index_lst, n):
     return index
 
 
-def load_yaml_template(filepath, n_dims):
-    with open(filepath) as f:
-        data = yaml.load(f.read())
+def load_yaml_templates(template_ids, filepaths, n_dims):
+    templates = {}
+    for template_id, fp in zip(template_ids, filepaths):
+        with open(fp) as f:
+            data = yaml.load(f.read())
+
+            if 'groups' in data:
+                raw_groups = data['groups']
+            else:
+                raise KeyError(
+                    "Missing 'groups' or 'template' key in yaml file %s"
+                    % fp)
 
-        if 'groups' in data:
-            raw_groups = data['groups']
-        else:
-            raise KeyError(
-                "Missing 'groups' or 'template' key in yaml file %s"
-                % filepath)
+            groups = []
 
-        groups = []
+            for index, group in enumerate(raw_groups):
 
-        for index, group in enumerate(raw_groups):
+                label = group.get('label', index)  # Allow simple ordered groups
 
-            label = group.get('label', index)  # Allow simple ordered groups
+                n = group['points']  # Should raise KeyError by design if missing
+                connectivity = group.get('connectivity', [])
 
-            n = group['points']  # Should raise KeyError by design if missing
-            connectivity = group.get('connectivity', [])
+                if isinstance(connectivity, list):
+                    index = parse_connectivity(connectivity, n)
+                elif connectivity == 'cycle':
+                    index = parse_connectivity(
+                        ['0:%d' % (n - 1), '%d 0' % (n - 1)], n)
+                else:
+                    index = []  # Couldn't parse connectivity, safe default
 
-            if isinstance(connectivity, list):
-                index = parse_connectivity(connectivity, n)
-            elif connectivity == 'cycle':
-                index = parse_connectivity(
-                    ['0:%d' % (n - 1), '%d 0' % (n - 1)], n)
-            else:
-                index = []  # Couldn't parse connectivity, safe default
+                groups.append(Group(label, n, index))
 
-            groups.append(Group(label, n, index))
+            templates[template_id] = groups
 
-    return build_json(groups, n_dims)
+    return build_json(templates, n_dims)
 
 
 def parse_group(group):
@@ -85,36 +89,34 @@ def group_to_json(group, n_dims):
     return group_json
 
 
-def build_json(groups, n_dims):
-    n_points = sum(g.n for g in groups)
-    offset = 0
-    connectivity = []
-    labels = []
-    for g in groups:
-        connectivity += [[j + offset for j in i] for i in g.index]
-        labels.append({
-            'label': g.label,
-            'mask': list(range(offset, offset + g.n))
-        })
-        offset += g.n
-
+def build_json(per_template_groups, n_dims):
     lm_json = {
-        'labels': labels,
-        'landmarks': {
-            'connectivity': connectivity,
-            'points': [[None] * n_dims] * n_points
-        },
-        'version': 2,
+        'groups': {},
+        'version': 3
     }
 
-    return lm_json
-
+    for template_id, groups in per_template_groups.iteritems():
+        n_points = sum(g.n for g in groups)
+        offset = 0
+        connectivity = []
+        labels = []
+        for g in groups:
+            connectivity += [[j + offset for j in i] for i in g.index]
+            labels.append({
+                'label': g.label,
+                'mask': list(range(offset, offset + g.n))
+            })
+            offset += g.n
+
+        lm_json['groups'][template_id] = {
+            'labels': labels,
+            'landmarks': {
+                'connectivity': connectivity,
+                'points': [[None] * n_dims] * n_points
+            }
+        }
 
-def load_legacy_template(path, n_dims):
-    with open(path) as f:
-        ta = f.read().strip().split('\n\n')
-    groups = [parse_group(g) for g in ta]
-    return build_json(groups, n_dims)
+    return lm_json
 
 
 def group_to_dict(g):
@@ -145,8 +147,8 @@ def convert_legacy_template(path):
     print " - {} > {} {}".format(path, new_path, warning)
 
 
-def load_template(path, n_dims):
-    return load_yaml_template(path, n_dims)
+def load_templates(lm_ids, filepaths, n_dims):
+    return load_yaml_templates(lm_ids, filepaths, n_dims)
 
 
 class TemplateAdapter(object):
@@ -162,9 +164,15 @@ def template_ids(self):
         pass
 
     @abc.abstractmethod
-    def load_template(self, lm_id):
+    def load_templates(self):
         pass
 
+    def repair_lm(self, lm_json):
+        t_ids = self.template_ids()
+        for t_id in t_ids:
+            if not t_id in lm_json['groups']:
+                lm_json['groups'][t_id] = self.load_templates()['groups'][t_id]
+
 
 class FileTemplateAdapter(TemplateAdapter):
 
@@ -211,9 +219,10 @@ def template_ids(self):
     def template_paths(self):
         return self.template_dir.glob('*' + FileExt.template)
 
-    def load_template(self, lm_id):
-        fp = safe_join(str(self.template_dir), lm_id + FileExt.template)
-        return load_template(fp, self.n_dims)
+    def load_templates(self):
+        template_ids = self.template_ids()
+        filepaths = [safe_join(str(self.template_dir), lm_id + FileExt.template) for lm_id in template_ids]
+        return load_templates(template_ids, filepaths, self.n_dims)
 
 
 class CachedFileTemplateAdapter(FileTemplateAdapter):
@@ -229,10 +238,9 @@ def __init__(self, n_dims, template_dir=None, upgrade_templates=False):
         FileTemplateAdapter.handle_old_templates(
             self, upgrade_templates=upgrade_templates)
 
-        self._cache = {lm_id: FileTemplateAdapter.load_template(self, lm_id)
-                       for lm_id in FileTemplateAdapter.template_ids(self)}
+        self._cache = FileTemplateAdapter.load_templates(self)
         print('cached {} templates ({})'.format(
-            len(self._cache), ', '.join(self._cache.keys())))
+            len(FileTemplateAdapter.template_ids(self)), ', '.join(FileTemplateAdapter.template_ids(self))))
 
-    def load_template(self, lm_id):
-        return self._cache[lm_id]
+    def load_templates(self):
+        return self._cache