From af3f3f9fbf542b6f77570701703ae92a54834c74 Mon Sep 17 00:00:00 2001 From: G Pugh Date: Fri, 5 Jul 2019 12:45:01 +0200 Subject: [PATCH 01/18] Added the ability to skip overwriting a smart group with the 'do_update=False' option. --- JSSImporter.py | 36 +++++++++++++++++++++++++++++------- 1 file changed, 29 insertions(+), 7 deletions(-) diff --git a/JSSImporter.py b/JSSImporter.py index 031e421..cf0ddd0 100644 --- a/JSSImporter.py +++ b/JSSImporter.py @@ -210,9 +210,10 @@ class JSSImporter(Processor): "Array of group dictionaries. Wrap each group in a " "dictionary. Group keys include 'name' (Name of the group to " "use, required), 'smart' (Boolean: static group=False, smart " - "group=True, default is False, not required), and " + "group=True, default is False, not required), " "template_path' (string: path to template file to use for " - "group, required for smart groups, invalid for static groups)", + "group, required for smart groups, invalid for static groups), " + "and 'do_update' (Boolean: default is True, not required)", }, "exclusion_groups": { "required": False, @@ -222,7 +223,8 @@ class JSSImporter(Processor): "use, required), 'smart' (Boolean: static group=False, smart " "group=True, default is False, not required), and " "template_path' (string: path to template file to use for " - "group, required for smart groups, invalid for static groups)", + "group, required for smart groups, invalid for static groups), " + "and 'do_update' (Boolean: default is True, not required)", }, "scripts": { "required": False, @@ -605,6 +607,7 @@ def handle_groups(self, groups): computer_groups = [] if groups: for group in groups: + self.output("Computer Group to process: %s" % group["name"]) if self.validate_input_var(group): is_smart = group.get("smart", False) if is_smart: @@ -856,6 +859,8 @@ def update_or_create_new(self, obj_cls, template_path, name="", update_env: The environment var to update if an object is updated. script_contents (str): XML escaped script. + do_update (Boolean): Do not overwrite an existing group if + set to False. Returns: The recipe object after updating. @@ -943,7 +948,7 @@ def get_templated_object(self, obj_cls, template_path): """Return an object based on a template located in search path. Args: - obj_cls: JSSObject class (for the purposes of JSSIMporter a + obj_cls: JSSObject class (for the purposes of JSSImporter a Policy or a ComputerGroup) template_path: String filename or path to template file. See find_file_in_search_path() for more information on @@ -1067,8 +1072,9 @@ def validate_input_var(self, var): # pylint: disable=no-self-use # Does the group have a blank value? (A blank value isn't really # invalid, but there's no need to process it further.) invalid = [False for value in var.values() if isinstance(value, str) - and (value.startswith("%") and value.endswith("%")) or not - value] + and (value.startswith("%") and value.endswith("%"))] + if not var.get('name') or not var.get('template_path'): + invalid = True return False if invalid else True def add_or_update_smart_group(self, group): @@ -1079,9 +1085,25 @@ def add_or_update_smart_group(self, group): self.replace_dict["site_id"] = group.get("site_id") if group.get("site_name"): self.replace_dict["site_name"] = group.get("site_name") + + # If do_update is set to False, do not update this object + do_update = group.get("do_update", True) + if not do_update: + try: + computer_group = self.jss.ComputerGroup(group["name"]) + self.output("Computer Group: %s already exists " + "and set not to update." % + computer_group.name) + return computer_group + except jss.GetError: + self.output("Computer Group: %s does not already exist. " + "Creating from template." % + group["name"]) + pass + computer_group = self.update_or_create_new( jss.ComputerGroup, group["template_path"], - update_env="jss_group_updated", added_env="jss_group_added") + update_env="jss_group_updated", added_env="jss_group_added", script_contents="") return computer_group From 2e4185e8e85ba8a4cf4938def915134105efd051 Mon Sep 17 00:00:00 2001 From: G Pugh Date: Fri, 12 Jul 2019 17:12:38 +0200 Subject: [PATCH 02/18] Added the ability to skip overwriting a smart group with the 'do_update=False' option. --- JSSImporter.py | 34 ++++++++++++++++++++++++++++------ 1 file changed, 28 insertions(+), 6 deletions(-) diff --git a/JSSImporter.py b/JSSImporter.py index 031e421..14b0a8e 100644 --- a/JSSImporter.py +++ b/JSSImporter.py @@ -210,9 +210,10 @@ class JSSImporter(Processor): "Array of group dictionaries. Wrap each group in a " "dictionary. Group keys include 'name' (Name of the group to " "use, required), 'smart' (Boolean: static group=False, smart " - "group=True, default is False, not required), and " + "group=True, default is False, not required), " "template_path' (string: path to template file to use for " - "group, required for smart groups, invalid for static groups)", + "group, required for smart groups, invalid for static groups), " + "and 'do_update' (Boolean: default is True, not required)", }, "exclusion_groups": { "required": False, @@ -222,7 +223,8 @@ class JSSImporter(Processor): "use, required), 'smart' (Boolean: static group=False, smart " "group=True, default is False, not required), and " "template_path' (string: path to template file to use for " - "group, required for smart groups, invalid for static groups)", + "group, required for smart groups, invalid for static groups), " + "and 'do_update' (Boolean: default is True, not required)", }, "scripts": { "required": False, @@ -605,6 +607,7 @@ def handle_groups(self, groups): computer_groups = [] if groups: for group in groups: + self.output("Computer Group to process: %s" % group["name"]) if self.validate_input_var(group): is_smart = group.get("smart", False) if is_smart: @@ -856,6 +859,8 @@ def update_or_create_new(self, obj_cls, template_path, name="", update_env: The environment var to update if an object is updated. script_contents (str): XML escaped script. + do_update (Boolean): Do not overwrite an existing group if + set to False. Returns: The recipe object after updating. @@ -943,7 +948,7 @@ def get_templated_object(self, obj_cls, template_path): """Return an object based on a template located in search path. Args: - obj_cls: JSSObject class (for the purposes of JSSIMporter a + obj_cls: JSSObject class (for the purposes of JSSImporter a Policy or a ComputerGroup) template_path: String filename or path to template file. See find_file_in_search_path() for more information on @@ -1067,8 +1072,9 @@ def validate_input_var(self, var): # pylint: disable=no-self-use # Does the group have a blank value? (A blank value isn't really # invalid, but there's no need to process it further.) invalid = [False for value in var.values() if isinstance(value, str) - and (value.startswith("%") and value.endswith("%")) or not - value] + and (value.startswith("%") and value.endswith("%"))] + if not var.get('name') or not var.get('template_path'): + invalid = True return False if invalid else True def add_or_update_smart_group(self, group): @@ -1079,6 +1085,22 @@ def add_or_update_smart_group(self, group): self.replace_dict["site_id"] = group.get("site_id") if group.get("site_name"): self.replace_dict["site_name"] = group.get("site_name") + + # If do_update is set to False, do not update this object + do_update = group.get("do_update", True) + if not do_update: + try: + computer_group = self.jss.ComputerGroup(group["name"]) + self.output("Computer Group: %s already exists " + "and set not to update." % + computer_group.name) + return computer_group + except jss.GetError: + self.output("Computer Group: %s does not already exist. " + "Creating from template." % + group["name"]) + pass + computer_group = self.update_or_create_new( jss.ComputerGroup, group["template_path"], update_env="jss_group_updated", added_env="jss_group_added") From 2cce58f0258e0f333e0ff8c61c68558d40a312dc Mon Sep 17 00:00:00 2001 From: G Pugh Date: Fri, 12 Jul 2019 17:14:29 +0200 Subject: [PATCH 03/18] change not required --- JSSImporter.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/JSSImporter.py b/JSSImporter.py index cf0ddd0..14b0a8e 100644 --- a/JSSImporter.py +++ b/JSSImporter.py @@ -1103,7 +1103,7 @@ def add_or_update_smart_group(self, group): computer_group = self.update_or_create_new( jss.ComputerGroup, group["template_path"], - update_env="jss_group_updated", added_env="jss_group_added", script_contents="") + update_env="jss_group_updated", added_env="jss_group_added") return computer_group From 8545dede9d7eb0d71ea5f3e48e10f172df60133d Mon Sep 17 00:00:00 2001 From: Nathaniel Strauss Date: Sat, 20 Jul 2019 18:30:32 -0500 Subject: [PATCH 04/18] Add support for skip_scope --- JSSImporter.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/JSSImporter.py b/JSSImporter.py index 031e421..4e189f6 100644 --- a/JSSImporter.py +++ b/JSSImporter.py @@ -288,6 +288,14 @@ class JSSImporter(Processor): "a PKG upload was not required since a PKG of the same name " "is already present on the server"), }, + "skip_scope": { + "required": False, + "default": False, + "description": + ("If True, policy scope will not be updated. By default group " + "and policy updates are coupled together. This allows group " + "updates without updating or overwriting policy scope."), + }, } output_variables = { "jss_changed_objects": { @@ -908,7 +916,10 @@ def update_or_create_new(self, obj_cls, template_path, name="", if not self.env.get('force_policy_state'): state = existing_object.find('general/enabled').text recipe_object.find('general/enabled').text = state - self.add_scope_to_policy(recipe_object) + + # If skip_scope is True then don't include scope data. + if self.env["skip_scope"] is not True: + self.add_scope_to_policy(recipe_object) self.add_scripts_to_policy(recipe_object) self.add_package_to_policy(recipe_object) From 0c7a12d2b29fa6e0fb9bb24bc0798f0fdbbb920f Mon Sep 17 00:00:00 2001 From: nstrauss Date: Mon, 29 Jul 2019 17:43:12 -0500 Subject: [PATCH 05/18] Add skip_scripts option --- JSSImporter.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/JSSImporter.py b/JSSImporter.py index 4e189f6..5105c76 100644 --- a/JSSImporter.py +++ b/JSSImporter.py @@ -296,6 +296,11 @@ class JSSImporter(Processor): "and policy updates are coupled together. This allows group " "updates without updating or overwriting policy scope."), }, + "skip_scripts": { + "required": False, + "default": False, + "description": "If True, policy scripts will not be updated.", + }, } output_variables = { "jss_changed_objects": { @@ -920,7 +925,8 @@ def update_or_create_new(self, obj_cls, template_path, name="", # If skip_scope is True then don't include scope data. if self.env["skip_scope"] is not True: self.add_scope_to_policy(recipe_object) - self.add_scripts_to_policy(recipe_object) + if self.env["skip_scripts"] is not True: + self.add_scripts_to_policy(recipe_object) self.add_package_to_policy(recipe_object) # If object is a script, add the passed contents to the object. From 627d332fa41c4a91a6e0bbc66c538a3130b3f971 Mon Sep 17 00:00:00 2001 From: Graham Date: Thu, 8 Aug 2019 22:41:16 +0200 Subject: [PATCH 06/18] fix for 409 when uploading pkg_object --- JSSImporter.py | 97 ++++++++++++++++++++++++++++---------------------- 1 file changed, 55 insertions(+), 42 deletions(-) diff --git a/JSSImporter.py b/JSSImporter.py index 031e421..39cb55f 100644 --- a/JSSImporter.py +++ b/JSSImporter.py @@ -431,6 +431,20 @@ def handle_category(self, category_type, category_name=None): # Category doesn't exist category = jss.Category(self.jss, category_name) category.save() + # wait for feedback that the category is there (only for cloud repos) + try: + self.env["JSS_REPOS"][0]["type"] + timeout = time.time() + 60 + while time.time() < timeout: + try: + category = self.jss.Category(category_name) + self.output("Category id: {}".format(category.id)) + break + except: + self.output("Waiting for category id from server...") + time.sleep(5) + except: + pass self.output( "Category type: %s-'%s' created." % (category_type, category_name)) @@ -478,9 +492,34 @@ def handle_package(self): pkg_update = (self.env["jss_changed_objects"]["jss_package_updated"]) except jss.GetError: # Package doesn't exist + self.output("Pkg-object does not exist according to JSS.") package = jss.Package(self.jss, self.pkg_name) pkg_update = (self.env["jss_changed_objects"]["jss_package_added"]) + os_requirements = self.env.get("os_requirements") + package_info = self.env.get("package_info") + package_notes = self.env.get("package_notes") + package_priority = self.env.get("package_priority") + package_reboot = self.env.get("package_reboot") + package_boot_volume_required = self.env.get( + "package_boot_volume_required") + + if self.category is not None: + cat_name = self.category.name + else: + cat_name = "" + self.update_object(cat_name, package, "category", pkg_update) + self.update_object(os_requirements, package, "os_requirements", + pkg_update) + self.update_object(package_info, package, "info", pkg_update) + self.update_object(package_notes, package, "notes", pkg_update) + self.update_object(package_priority, package, "priority", + pkg_update) + self.update_object(package_reboot, package, "reboot_required", + pkg_update) + self.update_object(package_boot_volume_required, package, + "boot_volume_required", pkg_update) + # Ensure packages are on distribution point(s) # If we had to make a new package object, we know we need to @@ -496,8 +535,24 @@ def handle_package(self): # will upload to the correct package object. Ignored by # AFP/SMB. if self.env["jss_changed_objects"]["jss_package_added"]: + # wait for feedback that the package is there (only for cloud repos) + try: + self.env["JSS_REPOS"][0]["type"] + timeout = time.time() + 60 + while time.time() < timeout: + try: + package.id + self.output("Uploaded package id: {}".format(package.id)) + break + except: + self.output("Waiting for package id from server...") + time.sleep(5) + except: + pass self.copy(pkg_path, id_=package.id) + self.upload_needed = True + # For AFP/SMB shares, we still want to see if the package # exists. If it's missing, copy it! elif not self.jss.distribution_points.exists( @@ -515,48 +570,6 @@ def handle_package(self): "is set to True.") self.env["stop_processing_recipe"] = True return - - # wait for feedback that the package is there (only for cloud repos) - try: - self.env["JSS_REPOS"][0]["type"] - timeout = time.time() + 60 - while time.time() < timeout: - try: - package = self.jss.Package(self.pkg_name) - break - except: - self.output("Waiting for package id from server...") - time.sleep(5) - self.output("Uploaded package id: {}".format(package.id)) - except: - pass - - pkg_update = ( - self.env["jss_changed_objects"]["jss_package_updated"]) - os_requirements = self.env.get("os_requirements") - package_info = self.env.get("package_info") - package_notes = self.env.get("package_notes") - package_priority = self.env.get("package_priority") - package_reboot = self.env.get("package_reboot") - package_boot_volume_required = self.env.get( - "package_boot_volume_required") - - if self.category is not None: - cat_name = self.category.name - else: - cat_name = "" - self.update_object(cat_name, package, "category", pkg_update) - self.update_object(os_requirements, package, "os_requirements", - pkg_update) - self.update_object(package_info, package, "info", pkg_update) - self.update_object(package_notes, package, "notes", pkg_update) - self.update_object(package_priority, package, "priority", - pkg_update) - self.update_object(package_reboot, package, "reboot_required", - pkg_update) - self.update_object(package_boot_volume_required, package, - "boot_volume_required", pkg_update) - else: package = None self.output("Package upload and object update skipped. If this is " From 1df95001c3f4382e30caa263c5766080662b0aad Mon Sep 17 00:00:00 2001 From: Graham Date: Thu, 8 Aug 2019 22:45:54 +0200 Subject: [PATCH 07/18] include timeout on all repo types, since some users reported 409 issues on local DPs --- JSSImporter.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/JSSImporter.py b/JSSImporter.py index af5c5d6..d4c03bf 100644 --- a/JSSImporter.py +++ b/JSSImporter.py @@ -441,9 +441,8 @@ def handle_category(self, category_type, category_name=None): # Category doesn't exist category = jss.Category(self.jss, category_name) category.save() - # wait for feedback that the category is there (only for cloud repos) + # wait for feedback that the category is there try: - self.env["JSS_REPOS"][0]["type"] timeout = time.time() + 60 while time.time() < timeout: try: @@ -545,9 +544,8 @@ def handle_package(self): # will upload to the correct package object. Ignored by # AFP/SMB. if self.env["jss_changed_objects"]["jss_package_added"]: - # wait for feedback that the package is there (only for cloud repos) + # wait for feedback that the package is there try: - self.env["JSS_REPOS"][0]["type"] timeout = time.time() + 60 while time.time() < timeout: try: From 2f2d65f4f72d9b6a8f67e7fa554db1834bbbc020 Mon Sep 17 00:00:00 2001 From: Graham Date: Thu, 8 Aug 2019 22:56:59 +0200 Subject: [PATCH 08/18] Version bump --- CHANGELOG.md | 23 ++++++++++++++++++----- JSSImporter.py | 2 +- pkg/jssimporter/build-info.plist | 2 +- version.plist | 2 +- 4 files changed, 21 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d9d8184..9b8c310 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,16 +3,29 @@ All notable changes to this project will be documented in this file. This project adheres to [Semantic Versioning](http://semver.org/). -## [1.0.2b4] - 2019-06-25 - 1.0.2b4 +### Known issues in latest version + + - `JCDS` mode remains "experimental" only. JCDS users may wish to continue to use the CDP mode if they encounter problems. + - Jamf cloud users may see intermittent failures of upload of packages, icons or other objects. We believe this is due to the clustering involved with Jamf Cloud Distribution Points. See (#81), (#119), (#145) etc. Ultimately, we need Jamf to provide proper endpoints for package uploads and managing icons. Please bug your Jamf support and sales consultants as often as possible! + + +## [1.0.2b5] - 2019-08-08 - 1.0.2b5 + +### Added + +- @grahamrpugh added a `do_update` feature to prevent overwriting a computer group if it already exists on the server, while continuing to create the group if it is not there. +- @nstrauss added a `skip_scope` feature to allow the upload of a policy without changing any existing scope. ### Fixed - - Minor update to embedded python-jss, which fixes a `urllib` problem when running in python2 (#151) + - Changed the order of the code which waits for the creation of a package id, and added a wait for the creation of a category id, to fix problems with package objects not yet existing when uploading a package. -### Known issues - - `JCDS` mode remains "experimental" only. JCDS users may wish to continue to use the CDP mode if they encounter problems. - - Jamf cloud users may see intermittent failures of upload of packages, icons or other objects. We believe this is due to the clustering involved with Jamf Cloud Distribution Points. See (#81), (#119), (#145) etc. Ultimately, we need Jamf to provide proper endpoints for package uploads and managing icons. Please bug your Jamf support and sales consultants as often as possible! +## [1.0.2b4] - 2019-06-25 - 1.0.2b4 + +### Fixed + + - Minor update to embedded python-jss, which fixes a `urllib` problem when running in python2 (#151) ## [1.0.2b3] - 2019-06-13 - A brave new world (with just a handful of men) diff --git a/JSSImporter.py b/JSSImporter.py index d4c03bf..9b20b7c 100644 --- a/JSSImporter.py +++ b/JSSImporter.py @@ -38,7 +38,7 @@ __all__ = ["JSSImporter"] -__version__ = "1.0.2b4" +__version__ = "1.0.2b5" REQUIRED_PYTHON_JSS_VERSION = StrictVersion("2.0.0") diff --git a/pkg/jssimporter/build-info.plist b/pkg/jssimporter/build-info.plist index 0489cf1..ed9543d 100644 --- a/pkg/jssimporter/build-info.plist +++ b/pkg/jssimporter/build-info.plist @@ -17,6 +17,6 @@ suppress_bundle_relocation version - 1.0.2b4 + 1.0.2b5 diff --git a/version.plist b/version.plist index 1f94922..c81c863 100644 --- a/version.plist +++ b/version.plist @@ -3,6 +3,6 @@ Version - 1.0.2b4 + 1.0.2b5 From 9a14d507b1094268c59f3bd315ca5748ec4ca518 Mon Sep 17 00:00:00 2001 From: Graham Date: Thu, 8 Aug 2019 23:08:02 +0200 Subject: [PATCH 09/18] Add skip_scripts to CHANGELOG --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9b8c310..b444b84 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,7 @@ All notable changes to this project will be documented in this file. This projec - @grahamrpugh added a `do_update` feature to prevent overwriting a computer group if it already exists on the server, while continuing to create the group if it is not there. - @nstrauss added a `skip_scope` feature to allow the upload of a policy without changing any existing scope. +- @nstrauss added a `skip_scripts` feature to allow the upload of a policy without changing any existing script objects in the script. ### Fixed From 8e0484b53a5f2a2a63ed39afa892ff1e92c1615a Mon Sep 17 00:00:00 2001 From: Graham Pugh Date: Fri, 16 Aug 2019 14:49:11 +0200 Subject: [PATCH 10/18] Added XML escape to fix category and Self Service description errors. --- .gitignore | 1 + JSSImporter.py | 8 ++++++-- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/.gitignore b/.gitignore index baf4526..e7d36e5 100644 --- a/.gitignore +++ b/.gitignore @@ -3,3 +3,4 @@ .ropeproject *.pkg *.DS_Store +local_tests/* diff --git a/JSSImporter.py b/JSSImporter.py index 69f139f..5d09fad 100644 --- a/JSSImporter.py +++ b/JSSImporter.py @@ -24,6 +24,7 @@ from distutils.version import StrictVersion from zipfile import ZipFile, ZIP_DEFLATED from xml.etree import ElementTree +from xml.sax.saxutils import escape sys.path.insert(0, '/Library/Application Support/JSSImporter') @@ -908,7 +909,7 @@ def update_or_create_new(self, obj_cls, template_path, name="", if policy_category is None: policy_category = ElementTree.SubElement( recipe_object, "category") - policy_category.text = self.env.get('policy_category') + policy_category.text = self.env.get('policy_category') if existing_object is not None: # If this policy already exists, and it has an icon set, @@ -1052,7 +1053,8 @@ def find_file_in_search_path(self, path): return final_path def replace_text(self, text, replace_dict): # pylint: disable=no-self-use - """Substitute items in a text string. + """Substitute items in a text string. Also escapes for XML, + as this is the only use for this definition Args: text: A string with embedded %tags%. @@ -1065,7 +1067,9 @@ def replace_text(self, text, replace_dict): # pylint: disable=no-self-use """ for key, value in replace_dict.iteritems(): # Wrap our keys in % to match template tags. + value = escape(value) text = text.replace("%%%s%%" % key, value) + return text def validate_input_var(self, var): # pylint: disable=no-self-use From 02c302437d31eea51045e8b6f967ef798dd38ee4 Mon Sep 17 00:00:00 2001 From: Graham Pugh Date: Thu, 22 Aug 2019 18:50:55 +0200 Subject: [PATCH 11/18] Reworking of handling packages on CDP-type instances to upload package first and then await feedback of the id. --- JSSImporter.py | 158 +++++++++++++++++++++++++++++-------------------- 1 file changed, 95 insertions(+), 63 deletions(-) diff --git a/JSSImporter.py b/JSSImporter.py index 71f74f7..0a33129 100644 --- a/JSSImporter.py +++ b/JSSImporter.py @@ -331,7 +331,7 @@ def __init__(self, env=None, infile=None, outfile=None): self.exclusion_groups = None self.scripts = None self.policy = None - self.upload_needed = True + self.upload_needed = False def main(self): """Main processor code.""" @@ -431,6 +431,17 @@ def init_jss_changed_objects(self): "jss_policy_added", "jss_policy_updated", "jss_icon_uploaded") self.env["jss_changed_objects"] = {key: [] for key in keys} + def repo_type(self): + """returns the type of repo""" + if self.env["JSS_REPOS"]: + try: + repo = self.env["JSS_REPOS"][0]["type"] + except KeyError: + repo = "DP" + else: + return + return repo + def handle_category(self, category_type, category_name=None): """Ensure a category is present.""" if self.env.get(category_type): @@ -485,7 +496,7 @@ def handle_package(self): """ # Skip package handling if there is no package or repos. pkg_path = self.env["pkg_path"] - if self.env["JSS_REPOS"] and pkg_path != "": + if self.repo_type() is not None and pkg_path != "": # Ensure that `pkg_path` is valid. if not os.path.exists(pkg_path): raise ProcessorError( @@ -495,27 +506,91 @@ def handle_package(self): if os.path.isdir(pkg_path): pkg_path = self.zip_pkg_path(pkg_path) self.env["pkg_path"] = pkg_path - # Make sure our change gets added back into the env for # visibility. self.pkg_name += ".zip" - + # now check if the package object already exists try: package = self.jss.Package(self.pkg_name) - self.output("Pkg-object already exists according to JSS, " - "moving on...") + self.output("Pkg-object already exists according to JSS.") + self.output("Package id: {}".format(package.id)) pkg_update = (self.env["jss_changed_objects"]["jss_package_updated"]) + # for cloud DPs we assume that the package object means there is an associated package except jss.GetError: # Package doesn't exist self.output("Pkg-object does not exist according to JSS.") - package = jss.Package(self.jss, self.pkg_name) - pkg_update = (self.env["jss_changed_objects"]["jss_package_added"]) + # for CDP or JDS types, the package has to be uploaded first to generate a package object + # then we wait for the package ID, then we can continue to assign attributes to the package + # object. + if self.repo_type() == "JDS" or self.repo_type() == "CDP" or self.repo_type() == "JCDS": + self.copy(pkg_path) + self.output("Package uploaded to {}.".format(self.repo_type())) + self.upload_needed = True + + # wait for feedback that the package is there + timeout = time.time() + 120 + while time.time() < timeout: + try: + package = self.jss.Package(self.pkg_name) + if package.id != 0: + self.output("Package id reported: {}".format(package.id)) + time.sleep(10) + break + else: + self.output("Waiting to get package id from cloud server (reported: {})...".format( + package.id)) + time.sleep(10) + except: + self.output("Waiting to get package id from cloud server (none reported)...") + time.sleep(10) + try: + package.id + except ValueError: + self.output("Failed to get package id from cloud server.") + self.env["stop_processing_recipe"] = True + return + pkg_update = (self.env["jss_changed_objects"]["jss_package_added"]) + elif self.repo_type() == "DP": + # for AFP/SMB shares, we create the package object first and then copy the package + # if it is not already there + package = jss.Package(self.jss, self.pkg_name) + else: + # no repo, or type that is not supported + self.output("Package not uploaded (repo type: {}).".format( + self.repo_type())) + self.upload_needed = False + + # For local DPs we check that the package is already on the distribution point and upload it if not + if self.repo_type() == "DP": + if not self.jss.distribution_points.exists( + os.path.basename(pkg_path)): + self.copy(pkg_path) + self.output("Package {} uploaded to distribution point.".format(self.pkg_name)) + self.upload_needed = True + else: + self.output("Package upload not required.") + self.upload_needed = False + + # only update the package object if an uploand ad was carried out + if (self.env["STOP_IF_NO_JSS_UPLOAD"] is True + and not self.upload_needed): + self.output("Not overwriting policy as upload requirement is determined as {} " + "and STOP_IF_NO_JSS_UPLOAD is set to {}.".format(self.upload_needed, + self.env["STOP_IF_NO_JSS_UPLOAD"])) + self.env["stop_processing_recipe"] = True + return + elif (self.env["STOP_IF_NO_JSS_UPLOAD"] is False + and not self.upload_needed): + self.output("Overwriting policy although upload requirement is determined as {} " + "because STOP_IF_NO_JSS_UPLOAD is set to {}.".format(self.upload_needed, + self.env["STOP_IF_NO_JSS_UPLOAD"])) + # now update the package object os_requirements = self.env.get("os_requirements") package_info = self.env.get("package_info") package_notes = self.env.get("package_notes") package_priority = self.env.get("package_priority") - package_reboot = self.env.get("package_reboot") + package_reboot = self.env.get("package_reboot") package_boot_volume_required = self.env.get( "package_boot_volume_required") @@ -535,55 +610,7 @@ def handle_package(self): self.update_object(package_boot_volume_required, package, "boot_volume_required", pkg_update) - # Ensure packages are on distribution point(s) - - # If we had to make a new package object, we know we need to - # copy the package file, regardless of DP type. This solves - # the issue regarding the JDS.exists() method: See - # python-jss docs for info. The problem with this method is - # that if you cancel an AutoPkg run and the package object - # has been created, but not uploaded, you will need to - # delete the package object from the JSS before running a - # recipe again or it won't upload the package file. - # - # Passes the id of the newly created package object so JDS' - # will upload to the correct package object. Ignored by - # AFP/SMB. - if self.env["jss_changed_objects"]["jss_package_added"]: - # wait for feedback that the package is there - try: - timeout = time.time() + 60 - while time.time() < timeout: - try: - package.id - self.output("Uploaded package id: {}".format(package.id)) - break - except: - self.output("Waiting for package id from server...") - time.sleep(5) - except: - pass - self.copy(pkg_path, id_=package.id) - - self.upload_needed = True - # For AFP/SMB shares, we still want to see if the package - # exists. If it's missing, copy it! - elif not self.jss.distribution_points.exists( - os.path.basename(pkg_path)): - self.copy(pkg_path) - self.upload_needed = True - else: - self.output("Package upload not needed.") - self.upload_needed = False - - # only update the package object if an upload was carried out - if (self.env["STOP_IF_NO_JSS_UPLOAD"] == True - and not self.upload_needed): - self.output("Not overwriting policy as STOP_IF_NO_JSS_UPLOAD " - "is set to True.") - self.env["stop_processing_recipe"] = True - return else: package = None self.output("Package upload and object update skipped. If this is " @@ -632,7 +659,7 @@ def handle_groups(self, groups): computer_groups = [] if groups: for group in groups: - self.output("Computer Group to process: %s" % group["name"]) + self.output("Computer Group to process: {}".format(group["name"])) if self.validate_input_var(group): is_smart = group.get("smart", False) if is_smart: @@ -679,10 +706,10 @@ def handle_policy(self): policy = self.update_or_create_new( jss.Policy, template_filename, update_env="jss_policy_updated", added_env="jss_policy_added") + self.output("Policy id: {}".format(policy.id)) else: self.output("Policy creation not desired, moving on...") policy = None - return policy def handle_icon(self): @@ -705,7 +732,6 @@ def handle_icon(self): # Search through search-paths for icon file. icon_path = self.find_file_in_search_path( self.env["self_service_icon"]) - icon_filename = os.path.basename(icon_path) # Compare the filename in the policy to the one provided by @@ -714,6 +740,7 @@ def handle_icon(self): policy_filename = self.policy.findtext( "self_service/self_service_icon/filename") if not policy_filename == icon_filename: + self.output("Icon name in existing policy: {}".format(policy_filename)) icon = jss.FileUpload(self.jss, "policies", "id", self.policy.id, icon_path) icon.save() @@ -942,8 +969,14 @@ def update_or_create_new(self, obj_cls, template_path, name="", # If skip_scope is True then don't include scope data. if self.env["skip_scope"] is not True: self.add_scope_to_policy(recipe_object) + else: + self.output("Skipping assignment of scope as skip_scope is set to {}".format(self.env["skip_scope"])) + # If skip_scripts is True then don't include scripts data. if self.env["skip_scripts"] is not True: self.add_scripts_to_policy(recipe_object) + else: + self.output("Skipping assignment of scripts as skip_scripts is set to {}".format(self.env["skip_scripts"])) + # add package to policy if there is one self.add_package_to_policy(recipe_object) # If object is a script, add the passed contents to the object. @@ -971,8 +1004,8 @@ def update_or_create_new(self, obj_cls, template_path, name="", self.env["jss_changed_objects"][added_env].append(name) return recipe_object - # pylint: enable=too-many-arguments + # pylint: enable=too-many-arguments def get_templated_object(self, obj_cls, template_path): """Return an object based on a template located in search path. @@ -1014,7 +1047,7 @@ def find_file_in_search_path(self, path): to copy templates, icons, etc, to the override directory. Args: - obj_cls: JSSObject class (for the purposes of JSSIMporter a + obj_cls: JSSObject class (for the purposes of JSSImporter a Policy or a ComputerGroup) path: String filename or path to file. @@ -1086,7 +1119,6 @@ def replace_text(self, text, replace_dict): # pylint: disable=no-self-use # Wrap our keys in % to match template tags. value = escape(value) text = text.replace("%%%s%%" % key, value) - return text def validate_input_var(self, var): # pylint: disable=no-self-use From a4600d662f7060c95f2fcbaf9c23282364b43904 Mon Sep 17 00:00:00 2001 From: grahampugh Date: Sat, 24 Aug 2019 14:37:20 +0200 Subject: [PATCH 12/18] Change all %s to format --- JSSImporter.py | 91 +++++++++++++++++++++++++++++--------------------- 1 file changed, 53 insertions(+), 38 deletions(-) diff --git a/JSSImporter.py b/JSSImporter.py index 0a33129..5a9260d 100644 --- a/JSSImporter.py +++ b/JSSImporter.py @@ -16,7 +16,6 @@ # limitations under the License. """See docstring for JSSImporter class.""" - import os import sys import time @@ -337,14 +336,14 @@ def main(self): """Main processor code.""" # Ensure we have the right version of python-jss python_jss_version = StrictVersion(PYTHON_JSS_VERSION) - self.output("python-jss version: %s." % python_jss_version) + self.output("python-jss version: {}.".format(python_jss_version)) if python_jss_version < REQUIRED_PYTHON_JSS_VERSION: self.output( - "python-jss version is too old. Please update to version: %s." - % REQUIRED_PYTHON_JSS_VERSION) + "python-jss version is too old. Please update to version: {}." + .format(REQUIRED_PYTHON_JSS_VERSION)) raise ProcessorError - self.output("JSSImporter version: %s." % __version__) + self.output("JSSImporter version: {}.".format(__version__)) # clear any pre-existing summary result if "jss_importer_summary_result" in self.env: @@ -359,8 +358,8 @@ def main(self): if self.version == "0.0.0.0": self.output( "Warning: No `version` was added to the AutoPkg env up to " - "this point. JSSImporter is defaulting to version %s!" - % self.version) + "this point. JSSImporter is defaulting to version {}!" + .format(self.version)) # Build and init jss_changed_objects self.init_jss_changed_objects() @@ -380,7 +379,7 @@ def main(self): self.package = self.handle_package() # stop if no package was uploaded and STOP_IF_NO_JSS_UPLOAD is True - if (self.env["STOP_IF_NO_JSS_UPLOAD"] == True + if (self.env["STOP_IF_NO_JSS_UPLOAD"] is True and not self.upload_needed): # Done with DPs, unmount them. for dp in self.jss.distribution_points: @@ -452,8 +451,8 @@ def handle_category(self, category_type, category_name=None): category = self.jss.Category(category_name) category_name = category.name self.output( - "Category type: %s-'%s' already exists according to JSS, " - "moving on..." % (category_type, category_name)) + "Category type '{}'-'{}' already exists according to JSS, " + "moving on...".format(category_type, category_name)) except jss.GetError: # Category doesn't exist category = jss.Category(self.jss, category_name) @@ -472,8 +471,8 @@ def handle_category(self, category_type, category_name=None): except: pass self.output( - "Category type: %s-'%s' created." % (category_type, - category_name)) + "Category type '{}'-'{}' created.".format(category_type, + category_name)) self.env["jss_changed_objects"]["jss_category_added"].append( category_name) else: @@ -523,10 +522,8 @@ def handle_package(self): # for CDP or JDS types, the package has to be uploaded first to generate a package object # then we wait for the package ID, then we can continue to assign attributes to the package # object. - if self.repo_type() == "JDS" or self.repo_type() == "CDP" or self.repo_type() == "JCDS": + if self.repo_type() == "JDS" or self.repo_type() == "CDP" or self.repo_type() == "AWS": self.copy(pkg_path) - self.output("Package uploaded to {}.".format(self.repo_type())) - self.upload_needed = True # wait for feedback that the package is there timeout = time.time() + 120 @@ -536,6 +533,8 @@ def handle_package(self): if package.id != 0: self.output("Package id reported: {}".format(package.id)) time.sleep(10) + self.output("Package uploaded to {}.".format(self.repo_type())) + self.upload_needed = True break else: self.output("Waiting to get package id from cloud server (reported: {})...".format( @@ -551,15 +550,22 @@ def handle_package(self): self.env["stop_processing_recipe"] = True return pkg_update = (self.env["jss_changed_objects"]["jss_package_added"]) - elif self.repo_type() == "DP": + elif self.repo_type() == "DP" or self.repo_type() == "Local": # for AFP/SMB shares, we create the package object first and then copy the package # if it is not already there package = jss.Package(self.jss, self.pkg_name) + pkg_update = (self.env["jss_changed_objects"]["jss_package_added"]) else: # no repo, or type that is not supported - self.output("Package not uploaded (repo type: {}).".format( - self.repo_type())) - self.upload_needed = False + if self.repo_type() is not None: + self.output("Package not uploaded. Repo type {} is not supported. " + "Please reconfigure your JSSImporter prefs.".format( + self.repo_type())) + self.env["stop_processing_recipe"] = True + return + else: + self.output("Package not uploaded as there are no repos.") + self.upload_needed = False # For local DPs we check that the package is already on the distribution point and upload it if not if self.repo_type() == "DP": @@ -635,7 +641,7 @@ def zip_pkg_path(self, path): for member in files: zip_handle.write(os.path.join(root, member)) - self.output("Closing: %s" % zip_name) + self.output("Closing: {}".format(zip_name)) return zip_name @@ -659,7 +665,7 @@ def handle_groups(self, groups): computer_groups = [] if groups: for group in groups: - self.output("Computer Group to process: {}".format(group["name"])) + # self.output("Computer Group to process: {}".format(group["name"])) if self.validate_input_var(group): is_smart = group.get("smart", False) if is_smart: @@ -837,23 +843,23 @@ def update_object(self, data, obj, path, update): if data != obj.findtext(path): obj.find(path).text = data obj.save() - self.output("%s %s updated." % ( + self.output("{} '{}' updated.".format( str(obj.__class__).split(".")[-1][:-2], path)) update.append(obj.name) def copy(self, source_item, id_=-1): """Copy a package or script using the JSS_REPOS preference.""" - self.output("Copying %s to all distribution points." % source_item) + self.output("Copying {} to all distribution points.".format(source_item)) def output_copy_status(connection): """Output AutoPkg copying status.""" - self.output("Copying to %s" % connection["url"]) + self.output("Copying to {}".format(connection["url"])) self.jss.distribution_points.copy(source_item, id_=id_, pre_callback=output_copy_status) self.env["jss_changed_objects"]["jss_repo_updated"].append( os.path.basename(source_item)) - self.output("Copied %s" % source_item) + self.output("Copied '{}'".format(source_item)) def build_replace_dict(self): """Build dict of replacement values based on available input.""" @@ -993,13 +999,13 @@ def update_or_create_new(self, obj_cls, template_path, name="", recipe_object.save() # Retrieve the updated XML. recipe_object = search_method(name) - self.output("%s: %s updated." % (obj_cls.__name__, name)) + self.output("{} '{}' updated.".format(obj_cls.__name__, name)) if update_env: self.env["jss_changed_objects"][update_env].append(name) else: # Object doesn't exist yet. recipe_object.save() - self.output("%s: %s created." % (obj_cls.__name__, name)) + self.output("{} '{}' created.".format(obj_cls.__name__, name)) if added_env: self.env["jss_changed_objects"][added_env].append(name) @@ -1092,13 +1098,13 @@ def find_file_in_search_path(self, path): tested.append(test_parent_folder_path) if final_path: - self.output("Found file: %s" % final_path) + self.output("Found file: {}".format(final_path)) break if not final_path: raise ProcessorError( - "Unable to find file %s at any of the following locations: %s" - % (filename, tested)) + "Unable to find file {} at any of the following locations: {}" + .format(filename, tested)) return final_path @@ -1155,19 +1161,28 @@ def add_or_update_smart_group(self, group): if not do_update: try: computer_group = self.jss.ComputerGroup(group["name"]) - self.output("Computer Group: %s already exists " + self.output("ComputerGroup '%s' already exists " "and set not to update." % computer_group.name) return computer_group except jss.GetError: - self.output("Computer Group: %s does not already exist. " + self.output("ComputerGroup '%s' does not already exist. " "Creating from template." % group["name"]) - pass computer_group = self.update_or_create_new( - jss.ComputerGroup, group["template_path"], - update_env="jss_group_updated", added_env="jss_group_added") + jss.ComputerGroup, group["template_path"], + update_env="jss_group_updated", added_env="jss_group_added") + # wait for feedback that the group is there + timeout = time.time() + 60 + while time.time() < timeout: + try: + group_check = self.jss.ComputerGroup(computer_group) + self.output("ComputerGroup id: {}".format(group_check.id)) + break + except: + self.output("Waiting for ComputerGroup id from server...") + time.sleep(5) return computer_group @@ -1176,12 +1191,12 @@ def add_or_update_static_group(self, group): # Check for pre-existing group first try: computer_group = self.jss.ComputerGroup(group["name"]) - self.output("Computer Group: %s already exists." % - computer_group.name) + self.output("Computer Group: {} already exists." + .format(computer_group.name)) except jss.GetError: computer_group = jss.ComputerGroup(self.jss, group["name"]) computer_group.save() - self.output("Computer Group: %s created." % computer_group.name) + self.output("Computer Group '{}' created.".format(computer_group.name)) self.env["jss_changed_objects"]["jss_group_added"].append( computer_group.name) From fb5e62c8214d71e5acb5ec106c091ebaf95ae2d8 Mon Sep 17 00:00:00 2001 From: Graham Pugh Date: Thu, 29 Aug 2019 17:18:28 +0200 Subject: [PATCH 13/18] Fixed an issue where the pkg_update variable was not assigned if there was no package upload. --- JSSImporter.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/JSSImporter.py b/JSSImporter.py index 5a9260d..59e60b4 100644 --- a/JSSImporter.py +++ b/JSSImporter.py @@ -499,10 +499,11 @@ def handle_package(self): # Ensure that `pkg_path` is valid. if not os.path.exists(pkg_path): raise ProcessorError( - "JSSImporter can't find a package at '%s'!" % pkg_path) + "JSSImporter can't find a package at '{}}'!".format(pkg_path)) # See if the package is non-flat (requires zipping prior to # upload). if os.path.isdir(pkg_path): + self.output("Pkg-object is a bundle. Converting to zip...") pkg_path = self.zip_pkg_path(pkg_path) self.env["pkg_path"] = pkg_path # Make sure our change gets added back into the env for @@ -517,7 +518,7 @@ def handle_package(self): # for cloud DPs we assume that the package object means there is an associated package except jss.GetError: # Package doesn't exist - self.output("Pkg-object does not exist according to JSS.") + self.output("Pkg-object does not already exist on the JSS.") # for CDP or JDS types, the package has to be uploaded first to generate a package object # then we wait for the package ID, then we can continue to assign attributes to the package @@ -545,14 +546,15 @@ def handle_package(self): time.sleep(10) try: package.id + pkg_update = (self.env["jss_changed_objects"]["jss_package_added"]) except ValueError: self.output("Failed to get package id from cloud server.") self.env["stop_processing_recipe"] = True return - pkg_update = (self.env["jss_changed_objects"]["jss_package_added"]) elif self.repo_type() == "DP" or self.repo_type() == "Local": # for AFP/SMB shares, we create the package object first and then copy the package # if it is not already there + self.output("Creating Pkg-object...") package = jss.Package(self.jss, self.pkg_name) pkg_update = (self.env["jss_changed_objects"]["jss_package_added"]) else: @@ -568,7 +570,7 @@ def handle_package(self): self.upload_needed = False # For local DPs we check that the package is already on the distribution point and upload it if not - if self.repo_type() == "DP": + if self.repo_type() == "DP" or self.repo_type() == "Local": if not self.jss.distribution_points.exists( os.path.basename(pkg_path)): self.copy(pkg_path) @@ -591,6 +593,7 @@ def handle_package(self): self.output("Overwriting policy although upload requirement is determined as {} " "because STOP_IF_NO_JSS_UPLOAD is set to {}.".format(self.upload_needed, self.env["STOP_IF_NO_JSS_UPLOAD"])) + # now update the package object os_requirements = self.env.get("os_requirements") package_info = self.env.get("package_info") From 0461fe37b8fefcb56ecdaade1491b849f96f3abc Mon Sep 17 00:00:00 2001 From: Graham Pugh Date: Thu, 29 Aug 2019 17:27:21 +0200 Subject: [PATCH 14/18] Bump version --- CHANGELOG.md | 10 ++++++++-- JSSImporter.py | 2 +- pkg/jssimporter/build-info.plist | 2 +- version.plist | 2 +- 4 files changed, 11 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b444b84..f12293f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,8 +5,14 @@ All notable changes to this project will be documented in this file. This projec ### Known issues in latest version - - `JCDS` mode remains "experimental" only. JCDS users may wish to continue to use the CDP mode if they encounter problems. - - Jamf cloud users may see intermittent failures of upload of packages, icons or other objects. We believe this is due to the clustering involved with Jamf Cloud Distribution Points. See (#81), (#119), (#145) etc. Ultimately, we need Jamf to provide proper endpoints for package uploads and managing icons. Please bug your Jamf support and sales consultants as often as possible! + - `JCDS` mode does not currently work. JCDS users should use the `CDP` mode. + - Efforts have been made to reduce incidences of this problem, but Jamf cloud users may see intermittent failures of upload of packages, icons or other objects. We believe this is due to the clustering involved with Jamf Cloud Distribution Points. See (#81), (#119), (#145) etc. Ultimately, we need Jamf to provide proper endpoints for package uploads and managing icons. Please bug your Jamf support and sales consultants as often as possible! + +## [1.0.2b6] - 2019-08-29 - 1.0.2b6 + +### Fixed + +- Fixed a bug that was introduced in 1.0.2b5 which prevented certain packages from uploading (relevant to #162). ## [1.0.2b5] - 2019-08-08 - 1.0.2b5 diff --git a/JSSImporter.py b/JSSImporter.py index 59e60b4..69d8f69 100644 --- a/JSSImporter.py +++ b/JSSImporter.py @@ -38,7 +38,7 @@ __all__ = ["JSSImporter"] -__version__ = "1.0.2b5" +__version__ = "1.0.2b6" REQUIRED_PYTHON_JSS_VERSION = StrictVersion("2.0.0") diff --git a/pkg/jssimporter/build-info.plist b/pkg/jssimporter/build-info.plist index ed9543d..f7df020 100644 --- a/pkg/jssimporter/build-info.plist +++ b/pkg/jssimporter/build-info.plist @@ -17,6 +17,6 @@ suppress_bundle_relocation version - 1.0.2b5 + 1.0.2b6 diff --git a/version.plist b/version.plist index c81c863..a38d0a2 100644 --- a/version.plist +++ b/version.plist @@ -3,6 +3,6 @@ Version - 1.0.2b5 + 1.0.2b6 From c4f64c4e3c2ab58b1285ea7279cdd5057ad4a4e2 Mon Sep 17 00:00:00 2001 From: Graham Pugh Date: Wed, 4 Sep 2019 11:57:35 +0200 Subject: [PATCH 15/18] Make clean now correctly removes payload. --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 1f5e161..70fbd48 100644 --- a/Makefile +++ b/Makefile @@ -49,5 +49,5 @@ $(PKG_ROOT)/Library/AutoPkg/autopkglib/JSSImporter.py: clean : @echo "Cleaning up package root" rm $(PKG_ROOT)/Library/AutoPkg/autopkglib/JSSImporter.py - rm -rf "$(PKG_ROOT)/Library/Application Support/JSSImporter/*" + rm -rf "$(PKG_ROOT)/Library/Application Support/JSSImporter/"* rm $(CURDIR)/pkg/jssimporter/build/*.pkg From fbe908ce132ca2fdddd34b21864585d324f42945 Mon Sep 17 00:00:00 2001 From: Graham Date: Sat, 14 Sep 2019 18:32:54 +0200 Subject: [PATCH 16/18] Added wait_for_id function. --- CHANGELOG.md | 15 +- JSSImporter.py | 641 ++++++++++++++++--------------- pkg/jssimporter/build-info.plist | 2 +- version.plist | 2 +- 4 files changed, 337 insertions(+), 323 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f12293f..dbd7970 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,8 +5,19 @@ All notable changes to this project will be documented in this file. This projec ### Known issues in latest version - - `JCDS` mode does not currently work. JCDS users should use the `CDP` mode. - - Efforts have been made to reduce incidences of this problem, but Jamf cloud users may see intermittent failures of upload of packages, icons or other objects. We believe this is due to the clustering involved with Jamf Cloud Distribution Points. See (#81), (#119), (#145) etc. Ultimately, we need Jamf to provide proper endpoints for package uploads and managing icons. Please bug your Jamf support and sales consultants as often as possible! + - `JCDS` mode does not currently work and will cause a recipe to fail if configured. JCDS users should use the `CDP` mode. + - Efforts continue to be made to reduce intermittent failures of upload of packages to Jamf Cloud Distribution Points and CDPs, icons or other objects, but they may still occur. We believe this is due to the clustering involved with Jamf Cloud Distribution Points. See (#81), (#119), (#145) etc. Ultimately, we need Jamf to provide proper endpoints for package uploads and managing icons. Please bug your Jamf support and sales consultants as often as possible! + - The above efforts to improve package upload reliability may conversely cause problems on setups with multiple DPs of different types. Scenarios involving Cloud plus Local DPs are not yet tested, and there probably needs to be a more intelligent method of treating each DP as a separate package upload process than currently exists. + + +## [1.0.2b7] - 2019-09-14 - 1.0.2b7 + +### Added + +- @grahamrpugh added a new `wait_for_id` definition, which provides a common method to check for feedback on the upload of each API object, in an attempt to reduce the chance of cloud clusters returning conflicting information about whether an object has been successfully uploaded or not. +- Verbosity is increased with respect to reporting object IDs. +- References to JSS are changed to "Jamf Pro Server"... except in the name `JSSImporter` of course! I think we're stuck with that one. + ## [1.0.2b6] - 2019-08-29 - 1.0.2b6 diff --git a/JSSImporter.py b/JSSImporter.py index 69d8f69..2a82340 100644 --- a/JSSImporter.py +++ b/JSSImporter.py @@ -38,8 +38,8 @@ __all__ = ["JSSImporter"] -__version__ = "1.0.2b6" -REQUIRED_PYTHON_JSS_VERSION = StrictVersion("2.0.0") +__version__ = "1.0.2b7" +REQUIRED_PYTHON_JSS_VERSION = StrictVersion("2.0.1") # pylint: disable=too-many-instance-attributes, too-many-public-methods @@ -109,7 +109,7 @@ class JSSImporter(Processor): "JSS_URL": { "required": True, "description": - "URL to a JSS that api the user has write access " + "URL to a Jamf Pro server that the API user has write access " "to, optionally set as a key in the com.github.autopkg " "preference file.", }, @@ -129,8 +129,8 @@ class JSSImporter(Processor): "JSS_VERIFY_SSL": { "required": False, "description": - "If set to False, SSL verification in communication" - " with the JSS will be skipped. Defaults to 'True'.", + "If set to False, SSL verification in communication " + "with the Jamf Pro server will be skipped. Defaults to 'True'.", "default": True, }, "JSS_SUPPRESS_WARNINGS": { @@ -160,7 +160,7 @@ class JSSImporter(Processor): "description": "If set to False JSSImporter will not override the policy " "enabled state. This allows creating new policies in a default " - "state and then going and manually enabling them in the JSS " + "state and then going and manually enabling them in the Jamf Pro server. " "Boolean, defaults to 'True'", "default": True, }, @@ -332,82 +332,6 @@ def __init__(self, env=None, infile=None, outfile=None): self.policy = None self.upload_needed = False - def main(self): - """Main processor code.""" - # Ensure we have the right version of python-jss - python_jss_version = StrictVersion(PYTHON_JSS_VERSION) - self.output("python-jss version: {}.".format(python_jss_version)) - if python_jss_version < REQUIRED_PYTHON_JSS_VERSION: - self.output( - "python-jss version is too old. Please update to version: {}." - .format(REQUIRED_PYTHON_JSS_VERSION)) - raise ProcessorError - - self.output("JSSImporter version: {}.".format(__version__)) - - # clear any pre-existing summary result - if "jss_importer_summary_result" in self.env: - del self.env["jss_importer_summary_result"] - - self.create_jss() - self.output("JSS version: '{}'".format(self.jss.version())) - - self.pkg_name = os.path.basename(self.env["pkg_path"]) - self.prod_name = self.env["prod_name"] - self.version = self.env.get("version") - if self.version == "0.0.0.0": - self.output( - "Warning: No `version` was added to the AutoPkg env up to " - "this point. JSSImporter is defaulting to version {}!" - .format(self.version)) - - # Build and init jss_changed_objects - self.init_jss_changed_objects() - - self.category = self.handle_category("category") - self.policy_category = self.handle_category("policy_category") - - # Get our DPs ready for copying. - if len(self.jss.distribution_points) == 0: - self.output("Warning: No distribution points configured!") - for dp in self.jss.distribution_points: - dp.was_mounted = hasattr(dp, 'is_mounted') and dp.is_mounted() - # Don't bother mounting the DPs if there's no package. - if self.env["pkg_path"]: - self.jss.distribution_points.mount() - - self.package = self.handle_package() - - # stop if no package was uploaded and STOP_IF_NO_JSS_UPLOAD is True - if (self.env["STOP_IF_NO_JSS_UPLOAD"] is True - and not self.upload_needed): - # Done with DPs, unmount them. - for dp in self.jss.distribution_points: - if not dp.was_mounted: - self.jss.distribution_points.umount() - self.summarize() - return - - # Build our text replacement dictionary - self.build_replace_dict() - - self.extattrs = self.handle_extension_attributes() - - self.groups = self.handle_groups(self.env.get("groups")) - self.exclusion_groups = self.handle_groups( - self.env.get("exclusion_groups")) - - self.scripts = self.handle_scripts() - self.policy = self.handle_policy() - self.handle_icon() - - # Done with DPs, unmount them. - for dp in self.jss.distribution_points: - if not dp.was_mounted: - self.jss.distribution_points.umount() - - self.summarize() - def create_jss(self): """Create a JSS object for API calls""" kwargs = { @@ -441,6 +365,27 @@ def repo_type(self): return return repo + def wait_for_id(self, obj_cls, obj_name): + """wait for feedback that the object is there""" + object = None + search_method = getattr(self.jss, obj_cls.__name__) + # limit time to wait to get a package ID. + timeout = time.time() + 120 + while time.time() < timeout: + try: + object = search_method(obj_name) + if object.id != 0: + self.output("{} ID '{}' verified on server".format(obj_cls.__name__, object.id)) + self.upload_needed = True + return object + else: + self.output("Waiting to get {} ID from server (reported: {})...".format(obj_cls.__name__, + object.id)) + time.sleep(10) + except jss.GetError: + self.output("Waiting to get {} ID from server (none reported)...".format(obj_cls.__name__)) + time.sleep(10) + def handle_category(self, category_type, category_name=None): """Ensure a category is present.""" if self.env.get(category_type): @@ -451,33 +396,24 @@ def handle_category(self, category_type, category_name=None): category = self.jss.Category(category_name) category_name = category.name self.output( - "Category type '{}'-'{}' already exists according to JSS, " + "Category, type '{}', name '{}', already exists on the Jamf Pro server, " "moving on...".format(category_type, category_name)) except jss.GetError: # Category doesn't exist category = jss.Category(self.jss, category_name) category.save() - # wait for feedback that the category is there + self.wait_for_id(jss.Category, category) try: - timeout = time.time() + 60 - while time.time() < timeout: - try: - category = self.jss.Category(category_name) - self.output("Category id: {}".format(category.id)) - break - except: - self.output("Waiting for category id from server...") - time.sleep(5) - except: - pass - self.output( - "Category type '{}'-'{}' created.".format(category_type, - category_name)) - self.env["jss_changed_objects"]["jss_category_added"].append( - category_name) + category.id + self.output( + "Category, type '{}', name '{}', created.".format(category_type, + category_name)) + self.env["jss_changed_objects"]["jss_category_added"].append( + category_name) + except ValueError: + raise ProcessorError("Failed to get category ID from {}.".format(self.repo_type())) else: category = None - return category def handle_package(self): @@ -495,135 +431,120 @@ def handle_package(self): """ # Skip package handling if there is no package or repos. pkg_path = self.env["pkg_path"] - if self.repo_type() is not None and pkg_path != "": - # Ensure that `pkg_path` is valid. - if not os.path.exists(pkg_path): - raise ProcessorError( - "JSSImporter can't find a package at '{}}'!".format(pkg_path)) - # See if the package is non-flat (requires zipping prior to - # upload). - if os.path.isdir(pkg_path): - self.output("Pkg-object is a bundle. Converting to zip...") - pkg_path = self.zip_pkg_path(pkg_path) - self.env["pkg_path"] = pkg_path - # Make sure our change gets added back into the env for - # visibility. - self.pkg_name += ".zip" - # now check if the package object already exists - try: - package = self.jss.Package(self.pkg_name) - self.output("Pkg-object already exists according to JSS.") - self.output("Package id: {}".format(package.id)) - pkg_update = (self.env["jss_changed_objects"]["jss_package_updated"]) - # for cloud DPs we assume that the package object means there is an associated package - except jss.GetError: - # Package doesn't exist - self.output("Pkg-object does not already exist on the JSS.") - - # for CDP or JDS types, the package has to be uploaded first to generate a package object - # then we wait for the package ID, then we can continue to assign attributes to the package - # object. - if self.repo_type() == "JDS" or self.repo_type() == "CDP" or self.repo_type() == "AWS": - self.copy(pkg_path) - - # wait for feedback that the package is there - timeout = time.time() + 120 - while time.time() < timeout: - try: - package = self.jss.Package(self.pkg_name) - if package.id != 0: - self.output("Package id reported: {}".format(package.id)) - time.sleep(10) - self.output("Package uploaded to {}.".format(self.repo_type())) - self.upload_needed = True - break - else: - self.output("Waiting to get package id from cloud server (reported: {})...".format( - package.id)) - time.sleep(10) - except: - self.output("Waiting to get package id from cloud server (none reported)...") - time.sleep(10) - try: - package.id - pkg_update = (self.env["jss_changed_objects"]["jss_package_added"]) - except ValueError: - self.output("Failed to get package id from cloud server.") - self.env["stop_processing_recipe"] = True - return - elif self.repo_type() == "DP" or self.repo_type() == "Local": - # for AFP/SMB shares, we create the package object first and then copy the package - # if it is not already there - self.output("Creating Pkg-object...") - package = jss.Package(self.jss, self.pkg_name) - pkg_update = (self.env["jss_changed_objects"]["jss_package_added"]) - else: - # no repo, or type that is not supported - if self.repo_type() is not None: - self.output("Package not uploaded. Repo type {} is not supported. " - "Please reconfigure your JSSImporter prefs.".format( - self.repo_type())) - self.env["stop_processing_recipe"] = True - return - else: - self.output("Package not uploaded as there are no repos.") - self.upload_needed = False - - # For local DPs we check that the package is already on the distribution point and upload it if not - if self.repo_type() == "DP" or self.repo_type() == "Local": - if not self.jss.distribution_points.exists( - os.path.basename(pkg_path)): - self.copy(pkg_path) - self.output("Package {} uploaded to distribution point.".format(self.pkg_name)) - self.upload_needed = True - else: - self.output("Package upload not required.") - self.upload_needed = False + if self.repo_type() is None or pkg_path == "": + self.output("Package upload and object update skipped. If this is " + "a mistake, ensure you have JSS_REPOS configured.") + return - # only update the package object if an uploand ad was carried out - if (self.env["STOP_IF_NO_JSS_UPLOAD"] is True - and not self.upload_needed): - self.output("Not overwriting policy as upload requirement is determined as {} " - "and STOP_IF_NO_JSS_UPLOAD is set to {}.".format(self.upload_needed, - self.env["STOP_IF_NO_JSS_UPLOAD"])) - self.env["stop_processing_recipe"] = True - return - elif (self.env["STOP_IF_NO_JSS_UPLOAD"] is False - and not self.upload_needed): - self.output("Overwriting policy although upload requirement is determined as {} " - "because STOP_IF_NO_JSS_UPLOAD is set to {}.".format(self.upload_needed, - self.env["STOP_IF_NO_JSS_UPLOAD"])) - - # now update the package object - os_requirements = self.env.get("os_requirements") - package_info = self.env.get("package_info") - package_notes = self.env.get("package_notes") - package_priority = self.env.get("package_priority") - package_reboot = self.env.get("package_reboot") - package_boot_volume_required = self.env.get( - "package_boot_volume_required") - - if self.category is not None: - cat_name = self.category.name + # Ensure that `pkg_path` is valid. + if not os.path.exists(pkg_path): + raise ProcessorError( + "JSSImporter can't find a package at '{}'!".format(pkg_path)) + + # See if the package is non-flat (requires zipping prior to + # upload). + if os.path.isdir(pkg_path): + self.output("Package object is a bundle. Converting to zip...") + pkg_path = self.zip_pkg_path(pkg_path) + self.env["pkg_path"] = pkg_path + # Make sure our change gets added back into the env for + # visibility. + self.pkg_name += ".zip" + + # now check if the package object already exists + try: + package = self.jss.Package(self.pkg_name) + self.output("Package object already exists on the Jamf Pro server. " + "(ID: {})".format(package.id)) + pkg_update = (self.env["jss_changed_objects"]["jss_package_updated"]) + # for cloud DPs we must assume that the package object means there is an associated package + except jss.GetError: + # Package doesn't exist + self.output("Package object does not already exist on the Jamf Pro server.") + + # for CDP or JDS types, the package has to be uploaded first to generate a package object + # then we wait for the package ID, then we can continue to assign attributes to the package + # object. + if self.repo_type() == "JDS" or self.repo_type() == "CDP" or self.repo_type() == "AWS": + self.copy(pkg_path) + package = self.wait_for_id(jss.Package, self.pkg_name) + try: + package.id + pkg_update = (self.env["jss_changed_objects"]["jss_package_added"]) + except ValueError: + raise ProcessorError("Failed to get Package ID from {}.".format(self.repo_type())) + + elif self.repo_type() == "DP" or self.repo_type() == "Local": + # for AFP/SMB shares, we create the package object first and then copy the package + # if it is not already there + self.output("Creating Package object...") + package = jss.Package(self.jss, self.pkg_name) + pkg_update = (self.env["jss_changed_objects"]["jss_package_added"]) else: - cat_name = "" - self.update_object(cat_name, package, "category", pkg_update) - self.update_object(os_requirements, package, "os_requirements", - pkg_update) - self.update_object(package_info, package, "info", pkg_update) - self.update_object(package_notes, package, "notes", pkg_update) - self.update_object(package_priority, package, "priority", - pkg_update) - self.update_object(package_reboot, package, "reboot_required", - pkg_update) - self.update_object(package_boot_volume_required, package, - "boot_volume_required", pkg_update) + # repo type that is not supported + raise ProcessorError( + "JSSImporter can't upload the Package at '{}'! Repo type {} is not supported. Please reconfigure your JSSImporter prefs.".format(pkg_path, self.repo_type())) + # For local DPs we check that the package is already on the distribution point and upload it if not + if self.repo_type() == "DP" or self.repo_type() == "Local": + if not self.jss.distribution_points.exists(os.path.basename(pkg_path)): + self.copy(pkg_path) + package = self.wait_for_id(jss.Package, self.pkg_name) + try: + package.id + pkg_update = (self.env["jss_changed_objects"]["jss_package_added"]) + except ValueError: + raise ProcessorError("Failed to get Package ID from {}.".format(self.repo_type())) + self.output("Package {} uploaded to distribution point.".format(self.pkg_name)) + self.upload_needed = True + else: + self.output("Package upload not required.") + self.upload_needed = False + # only update the package object if an uploand ad was carried out + if (self.env["STOP_IF_NO_JSS_UPLOAD"] is True + and not self.upload_needed): + self.output("Not overwriting policy as upload requirement is determined as {} " + "and STOP_IF_NO_JSS_UPLOAD is set to {}.".format(self.upload_needed, + self.env["STOP_IF_NO_JSS_UPLOAD"])) + self.env["stop_processing_recipe"] = True + return + elif (self.env["STOP_IF_NO_JSS_UPLOAD"] is False + and not self.upload_needed): + self.output("Overwriting policy although upload requirement is determined as {} " + "because STOP_IF_NO_JSS_UPLOAD is set to {}.".format(self.upload_needed, + self.env["STOP_IF_NO_JSS_UPLOAD"])) + + # now update the package object + os_requirements = self.env.get("os_requirements") + package_info = self.env.get("package_info") + package_notes = self.env.get("package_notes") + package_priority = self.env.get("package_priority") + package_reboot = self.env.get("package_reboot") + package_boot_volume_required = self.env.get( + "package_boot_volume_required") + + if self.category is not None: + cat_name = self.category.name else: - package = None - self.output("Package upload and object update skipped. If this is " - "a mistake, ensure you have JSS_REPOS configured.") + cat_name = "" + self.wait_for_id(jss.Package, self.pkg_name) + try: + package.id + pkg_update = (self.env["jss_changed_objects"]["jss_package_added"]) + except ValueError: + raise ProcessorError("Failed to get Package ID from {}.".format(self.repo_type())) + self.update_object(cat_name, package, "category", pkg_update) + self.update_object(os_requirements, package, "os_requirements", + pkg_update) + self.update_object(package_info, package, "info", pkg_update) + self.update_object(package_notes, package, "notes", pkg_update) + self.update_object(package_priority, package, "priority", + pkg_update) + self.update_object(package_reboot, package, "reboot_required", + pkg_update) + self.update_object(package_boot_volume_required, package, + "boot_volume_required", pkg_update) return package def zip_pkg_path(self, path): @@ -687,6 +608,7 @@ def handle_scripts(self): results = [] if scripts: for script in scripts: + self.output('Looking for Script file {}...'.format(script["name"])) script_file = self.find_file_in_search_path( script["name"]) try: @@ -694,7 +616,7 @@ def handle_scripts(self): script_contents = script_handle.read() except IOError: raise ProcessorError( - "Script '%s' could not be read!" % script_file) + "Script '{}' could not be read!".format(script_file)) script_object = self.update_or_create_new( jss.Script, @@ -715,7 +637,7 @@ def handle_policy(self): policy = self.update_or_create_new( jss.Policy, template_filename, update_env="jss_policy_updated", added_env="jss_policy_added") - self.output("Policy id: {}".format(policy.id)) + # self.output("PolicyPackage object: {}".format(policy.id)) else: self.output("Policy creation not desired, moving on...") policy = None @@ -725,7 +647,7 @@ def handle_icon(self): """Add self service icon if needed.""" # Icons are tricky. The only way to add new ones is to use # FileUploads. If you manually upload them, you can add them to - # a policy to get their ID, but there is no way to query the JSS + # a policy to get their ID, but there is no way to query the Jamf Pro server # to see what icons are available. Thus, icon handling involves # several cooperating methods. If we just add an icon every # time we run a recipe, however, we end up with a ton of @@ -739,6 +661,7 @@ def handle_icon(self): # If no policy handling is desired, we can't upload an icon. if self.env.get("self_service_icon") and self.policy is not None: # Search through search-paths for icon file. + self.output('Looking for Icon file {}...'.format(self.env["self_service_icon"])) icon_path = self.find_file_in_search_path( self.env["self_service_icon"]) icon_filename = os.path.basename(icon_path) @@ -755,80 +678,10 @@ def handle_icon(self): icon.save() self.env["jss_changed_objects"]["jss_icon_uploaded"].append( icon_filename) - self.output("Icon uploaded to JSS.") + self.output("Icon uploaded to the Jamf Pro server.") else: self.output("Icon matches existing icon, moving on...") - def summarize(self): - """If anything has been added or updated, report back.""" - # Only summarize if something has happened. - if any(value for value in self.env["jss_changed_objects"].values()): - # Create a blank summary. - self.env["jss_importer_summary_result"] = { - "summary_text": "The following changes were made to the JSS:", - "report_fields": [ - "Name", "Package", "Categories", "Groups", "Scripts", - "Extension_Attributes", "Policy", "Icon", "Version", - "Package_Uploaded"], - "data": { - "Name": "", - "Package": "", - "Categories": "", - "Groups": "", - "Scripts": "", - "Extension_Attributes": "", - "Policy": "", - "Icon": "", - "Version": "", - "Package_Uploaded": "" - } - } - # TODO: This is silly. Use a defaultdict for storing changes - # and just copy the stuff that changed. - - # Shortcut variables for lovely code conciseness - changes = self.env["jss_changed_objects"] - data = self.env["jss_importer_summary_result"]["data"] - - data["Name"] = self.env.get('NAME', '') - data["Version"] = self.env.get('version', '') - - package = self.get_report_string(changes["jss_package_added"] + - changes["jss_package_updated"]) - if package: - data["Package"] = package - - policy = changes["jss_policy_updated"] + ( - changes["jss_policy_added"]) - if policy: - data["Policy"] = self.get_report_string(policy) - - if changes["jss_icon_uploaded"]: - data["Icon"] = os.path.basename(self.env["self_service_icon"]) - - # Get nice strings for our list-types. - if changes["jss_category_added"]: - data["Categories"] = self.get_report_string( - changes["jss_category_added"]) - - groups = changes["jss_group_updated"] + changes["jss_group_added"] - if groups: - data["Groups"] = self.get_report_string(groups) - - scripts = changes["jss_script_updated"] + ( - changes["jss_script_added"]) - if scripts: - data["Scripts"] = self.get_report_string(scripts) - - extattrs = changes["jss_extension_attribute_updated"] + ( - changes["jss_extension_attribute_added"]) - if extattrs: - data["Extension_Attributes"] = self.get_report_string(extattrs) - - jss_package_uploaded = self.get_report_string(changes["jss_repo_updated"]) - if jss_package_uploaded: - data["Package_Uploaded"] = "True" - def update_object(self, data, obj, path, update): """Update an object if it differs. @@ -1000,17 +853,30 @@ def update_or_create_new(self, obj_cls, template_path, name="", # that it knows how to save itself. recipe_object._basic_identity["id"] = existing_object.id recipe_object.save() - # Retrieve the updated XML. - recipe_object = search_method(name) - self.output("{} '{}' updated.".format(obj_cls.__name__, name)) - if update_env: - self.env["jss_changed_objects"][update_env].append(name) + # get feedback that the object has been created + object = self.wait_for_id(obj_cls, name) + try: + object.id + # Retrieve the updated XML. + recipe_object = search_method(name) + self.output("{} '{}' updated.".format(obj_cls.__name__, name)) + if update_env: + self.env["jss_changed_objects"][update_env].append(name) + except ValueError: + raise ProcessorError("Failed to get {} ID from {}.".format(obj_cls.__name__, self.repo_type())) + else: # Object doesn't exist yet. recipe_object.save() - self.output("{} '{}' created.".format(obj_cls.__name__, name)) - if added_env: - self.env["jss_changed_objects"][added_env].append(name) + # get feedback that the object has been created + object = self.wait_for_id(obj_cls, name) + try: + object.id + self.output("{} '{}' created.".format(obj_cls.__name__, name)) + if added_env: + self.env["jss_changed_objects"][added_env].append(name) + except ValueError: + raise ProcessorError("Failed to get {} ID from {}.".format(obj_cls.__name__, self.repo_type())) return recipe_object @@ -1029,6 +895,7 @@ def get_templated_object(self, obj_cls, template_path): A JSS Object created based on the template, post-text-replacement. """ + self.output('Looking for {} template file {}...'.format(obj_cls.__name__, os.path.basename(template_path))) final_template_path = self.find_file_in_search_path(template_path) # Open and return a new object. @@ -1176,17 +1043,6 @@ def add_or_update_smart_group(self, group): computer_group = self.update_or_create_new( jss.ComputerGroup, group["template_path"], update_env="jss_group_updated", added_env="jss_group_added") - # wait for feedback that the group is there - timeout = time.time() + 60 - while time.time() < timeout: - try: - group_check = self.jss.ComputerGroup(computer_group) - self.output("ComputerGroup id: {}".format(group_check.id)) - break - except: - self.output("Waiting for ComputerGroup id from server...") - time.sleep(5) - return computer_group def add_or_update_static_group(self, group): @@ -1250,9 +1106,156 @@ def ensure_xml_structure(self, element, path): return element def get_report_string(self, items): # pylint: disable=no-self-use - """Return human-readable string from a list of JSS objects.""" + """Return human-readable string from a list of Jamf Pro API objects.""" return ", ".join(set(items)) + def summarize(self): + """If anything has been added or updated, report back.""" + # Only summarize if something has happened. + if any(value for value in self.env["jss_changed_objects"].values()): + # Create a blank summary. + self.env["jss_importer_summary_result"] = { + "summary_text": "The following changes were made to the Jamf Pro Server:", + "report_fields": [ + "Name", "Package", "Categories", "Groups", "Scripts", + "Extension_Attributes", "Policy", "Icon", "Version", + "Package_Uploaded"], + "data": { + "Name": "", + "Package": "", + "Categories": "", + "Groups": "", + "Scripts": "", + "Extension_Attributes": "", + "Policy": "", + "Icon": "", + "Version": "", + "Package_Uploaded": "" + } + } + # TODO: This is silly. Use a defaultdict for storing changes + # and just copy the stuff that changed. + + # Shortcut variables for lovely code conciseness + changes = self.env["jss_changed_objects"] + data = self.env["jss_importer_summary_result"]["data"] + + data["Name"] = self.env.get('NAME', '') + data["Version"] = self.env.get('version', '') + + package = self.get_report_string(changes["jss_package_added"] + + changes["jss_package_updated"]) + if package: + data["Package"] = package + + policy = changes["jss_policy_updated"] + ( + changes["jss_policy_added"]) + if policy: + data["Policy"] = self.get_report_string(policy) + + if changes["jss_icon_uploaded"]: + data["Icon"] = os.path.basename(self.env["self_service_icon"]) + + # Get nice strings for our list-types. + if changes["jss_category_added"]: + data["Categories"] = self.get_report_string( + changes["jss_category_added"]) + + groups = changes["jss_group_updated"] + changes["jss_group_added"] + if groups: + data["Groups"] = self.get_report_string(groups) + + scripts = changes["jss_script_updated"] + ( + changes["jss_script_added"]) + if scripts: + data["Scripts"] = self.get_report_string(scripts) + + extattrs = changes["jss_extension_attribute_updated"] + ( + changes["jss_extension_attribute_added"]) + if extattrs: + data["Extension_Attributes"] = self.get_report_string(extattrs) + + jss_package_uploaded = self.get_report_string(changes["jss_repo_updated"]) + if jss_package_uploaded: + data["Package_Uploaded"] = "True" + + def main(self): + """Main processor code.""" + # Ensure we have the right version of python-jss + python_jss_version = StrictVersion(PYTHON_JSS_VERSION) + self.output("python-jss version: {}.".format(python_jss_version)) + if python_jss_version < REQUIRED_PYTHON_JSS_VERSION: + self.output( + "python-jss version is too old. Please update to version: {}." + .format(REQUIRED_PYTHON_JSS_VERSION)) + raise ProcessorError + + self.output("JSSImporter version: {}.".format(__version__)) + + # clear any pre-existing summary result + if "jss_importer_summary_result" in self.env: + del self.env["jss_importer_summary_result"] + + self.create_jss() + self.output("Jamf Pro version: '{}'".format(self.jss.version())) + + self.pkg_name = os.path.basename(self.env["pkg_path"]) + self.prod_name = self.env["prod_name"] + self.version = self.env.get("version") + if self.version == "0.0.0.0": + self.output( + "Warning: No `version` was added to the AutoPkg env up to " + "this point. JSSImporter is defaulting to version {}!" + .format(self.version)) + + # Build and init jss_changed_objects + self.init_jss_changed_objects() + + self.category = self.handle_category("category") + self.policy_category = self.handle_category("policy_category") + + # Get our DPs ready for copying. + if len(self.jss.distribution_points) == 0: + self.output("Warning: No distribution points configured!") + for dp in self.jss.distribution_points: + self.output("Checking if DP already mounted...") + dp.was_mounted = hasattr(dp, 'is_mounted') and dp.is_mounted() + # Don't bother mounting the DPs if there's no package. + if self.env["pkg_path"]: + self.jss.distribution_points.mount() + + self.package = self.handle_package() + + # stop if no package was uploaded and STOP_IF_NO_JSS_UPLOAD is True + if (self.env["STOP_IF_NO_JSS_UPLOAD"] is True + and not self.upload_needed): + # Done with DPs, unmount them. + for dp in self.jss.distribution_points: + if not dp.was_mounted: + self.output("Unmounting DP...") + self.jss.distribution_points.umount() + self.summarize() + return + + # Build our text replacement dictionary + self.build_replace_dict() + + self.extattrs = self.handle_extension_attributes() + + self.groups = self.handle_groups(self.env.get("groups")) + self.exclusion_groups = self.handle_groups( + self.env.get("exclusion_groups")) + + self.scripts = self.handle_scripts() + self.policy = self.handle_policy() + self.handle_icon() + + # Done with DPs, unmount them. + for dp in self.jss.distribution_points: + if not dp.was_mounted: + self.jss.distribution_points.umount() + self.summarize() + # pylint: enable=too-many-instance-attributes, too-many-public-methods if __name__ == "__main__": diff --git a/pkg/jssimporter/build-info.plist b/pkg/jssimporter/build-info.plist index f7df020..931deb1 100644 --- a/pkg/jssimporter/build-info.plist +++ b/pkg/jssimporter/build-info.plist @@ -17,6 +17,6 @@ suppress_bundle_relocation version - 1.0.2b6 + 1.0.2b7 diff --git a/version.plist b/version.plist index a38d0a2..50ee0e7 100644 --- a/version.plist +++ b/version.plist @@ -3,6 +3,6 @@ Version - 1.0.2b6 + 1.0.2b7 From ebc882b62098badf7b9028b2cd33e1dba007f22a Mon Sep 17 00:00:00 2001 From: Graham Date: Tue, 17 Sep 2019 09:12:51 +0200 Subject: [PATCH 17/18] remove obsolete comments --- JSSImporter.py | 2 -- Makefile | 2 -- 2 files changed, 4 deletions(-) diff --git a/JSSImporter.py b/JSSImporter.py index 2a82340..d19f07b 100644 --- a/JSSImporter.py +++ b/JSSImporter.py @@ -923,8 +923,6 @@ def find_file_in_search_path(self, path): to copy templates, icons, etc, to the override directory. Args: - obj_cls: JSSObject class (for the purposes of JSSImporter a - Policy or a ComputerGroup) path: String filename or path to file. If path is just a filename, path is assumed to diff --git a/Makefile b/Makefile index 70fbd48..d2a36f1 100644 --- a/Makefile +++ b/Makefile @@ -40,8 +40,6 @@ $(PKG_ROOT)/Library/AutoPkg/autopkglib/JSSImporter.py: "$(PKG_ROOT)/Library/Application Support/JSSImporter/jss": @echo "Installing python-jss" - #@echo "Using amended PYTHONPATH inside package root, otherwise easy_install will complain we arent installing to a PYTHONPATH" - #cd $(JSS_GIT_ROOT) && PYTHONPATH="$(PKG_ROOT)/Library/Application Support/JSSImporter" easy_install --install-dir "$(PKG_ROOT)/Library/Application Support/JSSImporter" . mkdir -p "$(PKG_ROOT)/Library/Application Support/JSSImporter" cp -Rf "$(JSS_GIT_ROOT)/jss" "$(PKG_ROOT)/Library/Application Support/JSSImporter" From df120ed41079d03e196262af9baa7b5aa912bbec Mon Sep 17 00:00:00 2001 From: Graham Pugh Date: Wed, 25 Sep 2019 14:44:21 +0200 Subject: [PATCH 18/18] Version bump to official 1.0.2 --- CHANGELOG.md | 42 ++++++-------------------------- JSSImporter.py | 6 ++--- pkg/jssimporter/build-info.plist | 2 +- version.plist | 2 +- 4 files changed, 12 insertions(+), 40 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index dbd7970..a12b4de 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,53 +10,25 @@ All notable changes to this project will be documented in this file. This projec - The above efforts to improve package upload reliability may conversely cause problems on setups with multiple DPs of different types. Scenarios involving Cloud plus Local DPs are not yet tested, and there probably needs to be a more intelligent method of treating each DP as a separate package upload process than currently exists. -## [1.0.2b7] - 2019-09-14 - 1.0.2b7 +## [1.0.2] - 2019-09-25 - 1.0.2 -### Added +This is the official 1.0.2 release, exactly the same as the former 1.0.2b8. - @grahamrpugh added a new `wait_for_id` definition, which provides a common method to check for feedback on the upload of each API object, in an attempt to reduce the chance of cloud clusters returning conflicting information about whether an object has been successfully uploaded or not. - Verbosity is increased with respect to reporting object IDs. - References to JSS are changed to "Jamf Pro Server"... except in the name `JSSImporter` of course! I think we're stuck with that one. - - -## [1.0.2b6] - 2019-08-29 - 1.0.2b6 - -### Fixed - -- Fixed a bug that was introduced in 1.0.2b5 which prevented certain packages from uploading (relevant to #162). - - -## [1.0.2b5] - 2019-08-08 - 1.0.2b5 - -### Added - - @grahamrpugh added a `do_update` feature to prevent overwriting a computer group if it already exists on the server, while continuing to create the group if it is not there. - @nstrauss added a `skip_scope` feature to allow the upload of a policy without changing any existing scope. - @nstrauss added a `skip_scripts` feature to allow the upload of a policy without changing any existing script objects in the script. - -### Fixed - - - Changed the order of the code which waits for the creation of a package id, and added a wait for the creation of a category id, to fix problems with package objects not yet existing when uploading a package. - - -## [1.0.2b4] - 2019-06-25 - 1.0.2b4 - -### Fixed - - - Minor update to embedded python-jss, which fixes a `urllib` problem when running in python2 (#151) - - -## [1.0.2b3] - 2019-06-13 - A brave new world (with just a handful of men) - -There are a bunch of small fixes and improvements in this release, plus a few known issues - we'll update this file as time allows. - -### Added - - @grahamrpugh added a feature that prevents policies from being overwritten if there is no new package to upload, called `STOP_IF_NO_JSS_UPLOAD`. It is enabled by default. To override this behaviour and force the processor to continue and overwrite policies etc., run your autopkg recipe with the `--key STOP_IF_NO_JSS_UPLOAD=False` parameter. +- @grahamrpugh contributed (#135) which prevents uploaded scripts from having certain special characters incorrectly escaped, namely `>`, `<` and `&`. ### Fixed -- @grahamrpugh contributed (#135) which prevents uploaded scripts from having certain special characters incorrectly escaped, namely `>`, `<` and `&`. +- Fixed a bug that prevented Types `AFP` and `SMB` from being accepted (was introduced in 1.0.2b5). +- Fixed a bug that was introduced in 1.0.2b5 which prevented certain packages from uploading (relevant to #162). +- Changed the order of the code which waits for the creation of a package id, and added a wait for the creation of a category id, to fix problems with package objects not yet existing when uploading a package. +- Updated the embedded python-jss, which fixes a `urllib` problem when running in python2 (#151) ## [1.0.2b2] - 2018-09-22 - Bundled Dependency Testing diff --git a/JSSImporter.py b/JSSImporter.py index d19f07b..c65049d 100644 --- a/JSSImporter.py +++ b/JSSImporter.py @@ -38,7 +38,7 @@ __all__ = ["JSSImporter"] -__version__ = "1.0.2b7" +__version__ = "1.0.2" REQUIRED_PYTHON_JSS_VERSION = StrictVersion("2.0.1") @@ -474,7 +474,7 @@ def handle_package(self): except ValueError: raise ProcessorError("Failed to get Package ID from {}.".format(self.repo_type())) - elif self.repo_type() == "DP" or self.repo_type() == "Local": + elif self.repo_type() == "DP" or self.repo_type() == "SMB" or self.repo_type() == "AFP" or self.repo_type() == "Local": # for AFP/SMB shares, we create the package object first and then copy the package # if it is not already there self.output("Creating Package object...") @@ -486,7 +486,7 @@ def handle_package(self): "JSSImporter can't upload the Package at '{}'! Repo type {} is not supported. Please reconfigure your JSSImporter prefs.".format(pkg_path, self.repo_type())) # For local DPs we check that the package is already on the distribution point and upload it if not - if self.repo_type() == "DP" or self.repo_type() == "Local": + if self.repo_type() == "DP" or self.repo_type() == "SMB" or self.repo_type() == "AFP" or self.repo_type() == "Local": if not self.jss.distribution_points.exists(os.path.basename(pkg_path)): self.copy(pkg_path) package = self.wait_for_id(jss.Package, self.pkg_name) diff --git a/pkg/jssimporter/build-info.plist b/pkg/jssimporter/build-info.plist index 931deb1..52ce604 100644 --- a/pkg/jssimporter/build-info.plist +++ b/pkg/jssimporter/build-info.plist @@ -17,6 +17,6 @@ suppress_bundle_relocation version - 1.0.2b7 + 1.0.2 diff --git a/version.plist b/version.plist index 50ee0e7..031c536 100644 --- a/version.plist +++ b/version.plist @@ -3,6 +3,6 @@ Version - 1.0.2b7 + 1.0.2