Merge tag 'v0.3.0'

0.3.0

CHANGELOG.MD

@@ -84,3 +84,17 @@
 
 ### Bug fixes
 * Correct the last interpolated direction
+
+## 0.3.0 (2018-10-31)
+
+### Features and improvements
+* Enable specification of a time offset in case of already geotagged images
+* Add better progress bars and information output in process
+* Support Blackvue videos with embedded gps data
+* Add a simple `download` command to download all the blurred imaged from Mapillary for a certain `import_path`
+* Support import of multiple videos 
+
+### Breaking changes
+* Argument `--video_file` was renamed to `--video_import_path` as directories of one or more videos can now be processed and uploaded. Note that even if one video is to be processed and uploaded, the directory of the video should be specified as the video import path and not the video file itself. 
+* Only the Image Description EXIF tag is overwritten with the mapillary image description which includes all the data obtained during the `process` command. If one would like the rest of the tags to be overwritten, an additional argument needs to be passed under advanced usage. Single specific tags can also be overwritten by additional specific corresponding arguments.
+

bin/mapillary_tools

@@ -7,6 +7,7 @@ from mapillary_tools import commands
 
 advanced = '--advanced' in sys.argv
 version = '--version' in sys.argv
+full_help = '--full_help' in sys.argv
 
 if version:
     print("")
@@ -22,6 +23,8 @@ parser.add_argument(
     '--advanced', help='Use the tools under an advanced level with additional arguments and tools available.', action='store_true', required=False, default=False)
 parser.add_argument(
     '--version', help='Print mapillary tools version.', action='store_true', required=False, default=False)
+parser.add_argument(
+    '--full_help', help='Print full help for all the available commands and their arguments.', action='store_true', required=False, default=False)
 
 subparsers = parser.add_subparsers(
     help="Please choose one of the available tools", dest='tool', metavar='tool')
@@ -44,6 +47,14 @@ for command in all_commands:
     if advanced:
         command.add_advanced_arguments(subparser)
 
+if full_help:
+    subparsers_actions = [action for action in parser._actions if isinstance(
+        action, argparse._SubParsersAction)]
+    for subparsers_action in subparsers_actions:
+        for choice, subparser in subparsers_action.choices.items():
+            print("Subcommand '{}'".format(choice))
+            print(subparser.format_help())
+
 args = parser.parse_args()
 
 args_command = vars(args)['tool']
@@ -52,6 +63,8 @@ if 'advanced' in vars(args):
     del vars(args)['advanced']
 if 'version' in vars(args):
     del vars(args)['version']
+if 'full_help' in vars(args):
+    del vars(args)['full_help']
 
 # Run the selected subcommand if unit command, or in case of batch
 # command, run several unit commands

mapillary_tools/commands/__init__.py

@@ -14,6 +14,7 @@
 from . import authenticate
 from . import interpolate
 from . import post_process
+from . import download
 
 mapillary_tools_advanced_commands = [
     sample_video,
@@ -28,7 +29,8 @@
     process_csv,
     authenticate,
     interpolate,
-    post_process
+    post_process,
+    download
 ]
 
 mapillary_tools_commands = [
@@ -37,7 +39,7 @@
     process_and_upload
 ]
 
-VERSION = "0.2.0"
+VERSION = "0.3.0"
 
 
 def add_general_arguments(parser, command):

mapillary_tools/commands/download.py

@@ -0,0 +1,24 @@
+import inspect
+from mapillary_tools.download import download
+
+
+class Command:
+    name = 'download'
+    help = 'Helper tool : For a specified import path, download the blurred images from Mapillary.'
+
+    def add_basic_arguments(self, parser):
+        parser.add_argument("--output_folder",
+                            help="Output folder for the downloaded images.", required=True)
+        parser.add_argument("--user_name", help="user name", required=True)
+
+    def add_advanced_arguments(self, parser):
+        parser.add_argument(
+            '--number_threads', help='Specify the number of download threads.', type=int, default=10, required=False)
+
+    def run(self, args):
+
+        vars_args = vars(args)
+        download(**({k: v for k, v in vars_args.iteritems()
+                     if k in inspect.getargspec(download).args}))
+
+        print("Download done.")

mapillary_tools/commands/exif_insert.py

@@ -18,7 +18,17 @@ def add_advanced_arguments(self, parser):
                             action='store_true', default=False, required=False)
         parser.add_argument('--skip_EXIF_insert', help='Skip inserting the extracted data into image EXIF.',
                             action='store_true', default=False, required=False)
-        parser.add_argument('--keep_original', help='Do not overwrite original images, instead save the processed images in a new directory by adding suffix "_processed" to the import_path.',
+        parser.add_argument('--keep_original', help='Do not overwrite original images, instead save the processed images in a new directory called "processed_images" located in .mapillary in the import_path.',
+                            action='store_true', default=False, required=False)
+        parser.add_argument('--overwrite_all_EXIF_tags', help='Overwrite the rest of the EXIF tags, whose values are changed during the processing. Default is False, which will result in the processed values to be inserted only in the EXIF Image Description tag.',
+                            action='store_true', default=False, required=False)
+        parser.add_argument('--overwrite_EXIF_time_tag', help='Overwrite the capture time EXIF tag with the value obtained in process.',
+                            action='store_true', default=False, required=False)
+        parser.add_argument('--overwrite_EXIF_gps_tag', help='Overwrite the gps EXIF tag with the value obtained in process.',
+                            action='store_true', default=False, required=False)
+        parser.add_argument('--overwrite_EXIF_direction_tag', help='Overwrite the camera direction EXIF tag with the value obtained in process.',
+                            action='store_true', default=False, required=False)
+        parser.add_argument('--overwrite_EXIF_orientation_tag', help='Overwrite the orientation EXIF tag with the value obtained in process.',
                             action='store_true', default=False, required=False)
 
     def run(self, args):

mapillary_tools/commands/extract_geotag_data.py

@@ -32,4 +32,6 @@ def add_advanced_arguments(self, parser):
                             help="Use GPS trace starting time in case of derivating timestamp from filename.", action="store_true", default=False, required=False)
 
     def run(self, args):
+        if args.geotag_source == 'blackvue_videos' and not args.device_make:
+            args.device_make = "Blackvue"
         process_geotag_properties(**vars(args))

mapillary_tools/commands/post_process.py

@@ -33,6 +33,8 @@ def add_advanced_arguments(self, parser):
                             action='store_true', default=False, required=False)
         parser.add_argument('--split_import_path', help='Provide the path where the images should be moved to based on the import status.',
                             action='store', required=False, default=None)
+        parser.add_argument('--save_local_mapping', help='Save the mapillary photo uuid to local file mapping in a csv.',
+                            action='store_true', default=False, required=False)
 
     def run(self, args):
 

mapillary_tools/commands/process.py

@@ -83,7 +83,7 @@ def add_advanced_arguments(self, parser):
         # EXIF insert
         parser.add_argument('--skip_EXIF_insert', help='Skip inserting the extracted data into image EXIF.',
                             action='store_true', default=False, required=False)
-        parser.add_argument('--keep_original', help='Do not overwrite original images, instead save the processed images in a new directory by adding suffix "_processed" to the import_path.',
+        parser.add_argument('--keep_original', help='Do not overwrite original images, instead save the processed images in a new directory called "processed_images" located in .mapillary in the import_path.',
                             action='store_true', default=False, required=False)
 
         # post process
@@ -99,7 +99,18 @@ def add_advanced_arguments(self, parser):
                             action='store_true', default=False, required=False)
         parser.add_argument(
             '--split_import_path', help='If splitting the import path into duplicates, sequences, success and failed uploads, provide a path for the splits.', default=None, required=False)
-
+        parser.add_argument('--save_local_mapping', help='Save the mapillary photo uuid to local file mapping in a csv.',
+                            action='store_true', default=False, required=False)
+        parser.add_argument('--overwrite_all_EXIF_tags', help='Overwrite the rest of the EXIF tags, whose values are changed during the processing. Default is False, which will result in the processed values to be inserted only in the EXIF Image Description tag.',
+                            action='store_true', default=False, required=False)
+        parser.add_argument('--overwrite_EXIF_time_tag', help='Overwrite the capture time EXIF tag with the value obtained in process.',
+                            action='store_true', default=False, required=False)
+        parser.add_argument('--overwrite_EXIF_gps_tag', help='Overwrite the gps EXIF tag with the value obtained in process.',
+                            action='store_true', default=False, required=False)
+        parser.add_argument('--overwrite_EXIF_direction_tag', help='Overwrite the camera direction EXIF tag with the value obtained in process.',
+                            action='store_true', default=False, required=False)
+        parser.add_argument('--overwrite_EXIF_orientation_tag', help='Overwrite the orientation EXIF tag with the value obtained in process.',
+                            action='store_true', default=False, required=False)
         # add custom meta data in a form of a string consisting of a triplet
         # "name,type,value"
         parser.add_argument('--custom_meta_data', help='Add custom meta data to all images. Required format of input is a string, consisting of the meta data name, type and value, separated by a comma for each entry, where entries are separated by semicolon. Supported types are long, double, string, boolean, date. Example for two meta data entries "random_name1,double,12.34;random_name2,long,1234"',
@@ -108,6 +119,8 @@ def add_advanced_arguments(self, parser):
     def run(self, args):
 
         vars_args = vars(args)
+        if args.geotag_source == 'blackvue_videos' and not args.device_make:
+            args.device_make = "Blackvue"
         process_user_properties(**({k: v for k, v in vars_args.iteritems()
                                     if k in inspect.getargspec(process_user_properties).args}))
 

mapillary_tools/commands/process_and_upload.py

@@ -26,8 +26,6 @@ def add_basic_arguments(self, parser):
             '--organization_key', help="Specify organization key", default=None, required=False)
         parser.add_argument('--private',
                             help="Specify whether the import is private", action='store_true', default=False, required=False)
-        parser.add_argument(
-            '--manual_done', help='Manually finalize the upload', action='store_true', default=False, required=False)
         parser.add_argument(
             '--skip_subfolders', help='Skip all subfolders and import only the images in the given directory path.', action='store_true', default=False, required=False)
 
@@ -53,7 +51,7 @@ def add_advanced_arguments(self, parser):
 
         # geotagging
         parser.add_argument('--geotag_source', help='Provide the source of date/time and gps information needed for geotagging.', action='store',
-                            choices=['exif', 'gpx', 'gopro_video', 'nmea'], default="exif", required=False)
+                            choices=['exif', 'gpx', 'gopro_video', 'nmea', 'blackvue_videos'], default="exif", required=False)
         parser.add_argument(
             '--geotag_source_path', help='Provide the path to the file source of date/time and gps information needed for geotagging.', action='store',
             default=None, required=False)
@@ -85,7 +83,7 @@ def add_advanced_arguments(self, parser):
         # EXIF insert
         parser.add_argument('--skip_EXIF_insert', help='Skip inserting the extracted data into image EXIF.',
                             action='store_true', default=False, required=False)
-        parser.add_argument('--keep_original', help='Do not overwrite original images, instead save the processed images in a new directory by adding suffix "_processed" to the import_path.',
+        parser.add_argument('--keep_original', help='Do not overwrite original images, instead save the processed images in a new directory called "processed_images" located in .mapillary in the import_path.',
                             action='store_true', default=False, required=False)
         parser.add_argument(
             '--number_threads', help='Specify the number of upload threads.', type=int, default=None, required=False)
@@ -105,7 +103,18 @@ def add_advanced_arguments(self, parser):
                             action='store_true', default=False, required=False)
         parser.add_argument(
             '--split_import_path', help='If splitting the import path into duplicates, sequences, success and failed uploads, provide a path for the splits.', default=None, required=False)
-
+        parser.add_argument('--save_local_mapping', help='Save the mapillary photo uuid to local file mapping in a csv.',
+                            action='store_true', default=False, required=False)
+        parser.add_argument('--overwrite_all_EXIF_tags', help='Overwrite the rest of the EXIF tags, whose values are changed during the processing. Default is False, which will result in the processed values to be inserted only in the EXIF Image Description tag.',
+                            action='store_true', default=False, required=False)
+        parser.add_argument('--overwrite_EXIF_time_tag', help='Overwrite the capture time EXIF tag with the value obtained in process.',
+                            action='store_true', default=False, required=False)
+        parser.add_argument('--overwrite_EXIF_gps_tag', help='Overwrite the gps EXIF tag with the value obtained in process.',
+                            action='store_true', default=False, required=False)
+        parser.add_argument('--overwrite_EXIF_direction_tag', help='Overwrite the camera direction EXIF tag with the value obtained in process.',
+                            action='store_true', default=False, required=False)
+        parser.add_argument('--overwrite_EXIF_orientation_tag', help='Overwrite the orientation EXIF tag with the value obtained in process.',
+                            action='store_true', default=False, required=False)
         # add custom meta data in a form of a string consisting of a triplet
         # "name,type,value"
         parser.add_argument('--custom_meta_data', help='Add custom meta data to all images. Required format of input is a string, consisting of the meta data name, type and value, separated by a comma for each entry, where entries are separated by semicolon. Supported types are long, double, string, boolean, date. Example for two meta data entries "random_name1,double,12.34;random_name2,long,1234"',
@@ -114,6 +123,8 @@ def add_advanced_arguments(self, parser):
     def run(self, args):
 
         vars_args = vars(args)
+        if args.geotag_source == 'blackvue_videos' and not args.device_make:
+            args.device_make = "Blackvue"
         process_user_properties(**({k: v for k, v in vars_args.iteritems()
                                     if k in inspect.getargspec(process_user_properties).args}))
 

mapillary_tools/commands/sample_video.py

@@ -8,7 +8,7 @@ class Command:
 
     def add_basic_arguments(self, parser):
         # video specific args
-        parser.add_argument('--video_file', help='Provide the path to a video file or a directory containing a set of Blackvue video files.',
+        parser.add_argument('--video_import_path', help='Path to a directory with one or more video files.',
                             action='store', required=True)
         parser.add_argument('--video_sample_interval',
                             help='Time interval for sampled video frames in seconds', default=2, type=float, required=False)

mapillary_tools/commands/upload.py

@@ -10,8 +10,6 @@ class Command:
     def add_basic_arguments(self, parser):
 
         # command specific args
-        parser.add_argument(
-            '--manual_done', help='Manually finalize the upload', action='store_true', default=False, required=False)
         parser.add_argument(
             '--skip_subfolders', help='Skip all subfolders and import only the images in the given directory path.', action='store_true', default=False, required=False)
 
@@ -32,6 +30,8 @@ def add_advanced_arguments(self, parser):
                             action='store_true', default=False, required=False)
         parser.add_argument('--push_images', help='Push images uploaded in given import path.',
                             action='store_true', default=False, required=False)
+        parser.add_argument('--save_local_mapping', help='Save the mapillary photo uuid to local file mapping in a csv.',
+                            action='store_true', default=False, required=False)
 
     def run(self, args):