From 7e61f515d9e345e2664492c89968854f877a4916 Mon Sep 17 00:00:00 2001
From: Wendell Piez <wendell.piez@nist.gov>
Date: Wed, 11 Sep 2024 17:21:18 -0400
Subject: [PATCH] Now with complete pipeline producing OSCAL from PDF via HTML
 and NISO STS intermediates

---
 projects/oscal-from-pdf/GRAB-FM6-22.xpl       |  20 ++
 projects/oscal-from-pdf/GRAB-RESOURCES.xpl    |  37 +++
 .../extract-FM6_22-chapter4.xpl               | 253 +++++++++++++-----
 projects/oscal-from-pdf/readme.md             |  40 +--
 .../oscal-from-pdf/src/fm22-6-html-to-sts.xsl |   2 +-
 .../src/fm22-6_chapter4-sts.sch               |  48 ++++
 .../oscal-from-pdf/src/fm22-6_chapter4.sch    |  13 -
 .../src/fm22-6_sts-enhance1.xsl               | 183 +++++++++++++
 .../src/fm22-6_sts-enhance2.xsl               | 143 ++++++++++
 .../src/fm22-6_sts-enhance3.xsl               |  39 +++
 .../src/fm22-6_sts-to-oscal.xsl               | 252 +++++++++++++++++
 projects/oscal-from-pdf/src/oscal-check.sch   |  18 ++
 .../oscal-from-pdf/src/xvrl-summarize.xpl     |  50 ++++
 13 files changed, 992 insertions(+), 106 deletions(-)
 create mode 100644 projects/oscal-from-pdf/GRAB-FM6-22.xpl
 create mode 100644 projects/oscal-from-pdf/GRAB-RESOURCES.xpl
 create mode 100644 projects/oscal-from-pdf/src/fm22-6_chapter4-sts.sch
 delete mode 100644 projects/oscal-from-pdf/src/fm22-6_chapter4.sch
 create mode 100644 projects/oscal-from-pdf/src/fm22-6_sts-enhance1.xsl
 create mode 100644 projects/oscal-from-pdf/src/fm22-6_sts-enhance2.xsl
 create mode 100644 projects/oscal-from-pdf/src/fm22-6_sts-enhance3.xsl
 create mode 100644 projects/oscal-from-pdf/src/fm22-6_sts-to-oscal.xsl
 create mode 100644 projects/oscal-from-pdf/src/oscal-check.sch
 create mode 100644 projects/oscal-from-pdf/src/xvrl-summarize.xpl

diff --git a/projects/oscal-from-pdf/GRAB-FM6-22.xpl b/projects/oscal-from-pdf/GRAB-FM6-22.xpl
new file mode 100644
index 00000000..72ffda49
--- /dev/null
+++ b/projects/oscal-from-pdf/GRAB-FM6-22.xpl
@@ -0,0 +1,20 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<p:declare-step xmlns:p="http://www.w3.org/ns/xproc"
+   xmlns:c="http://www.w3.org/ns/xproc-step" version="3.0"
+   xmlns:ox="http://csrc.nist.gov/ns/oscal-xproc3"
+   type="ox:GRAB-FM6-22"
+   name="GRAB-FM6-22">
+   
+   <!-- =+=+=+=+= HOUSE RULES HALL PASS - remove this comment when the file is committed to the repo =+=+=+=+=  -->
+
+  <!-- For samples and boilerplate see file ../projects/xproc-doc/xproc-snippets.xml -->
+
+<!-- https://rdl.train.army.mil/catalog-ws/view/100.ATSC/3110F413-E915-47C1-85AA-DD4065C654C3-1274570636396/fm6_22.pdf -->
+   
+   <p:variable name="FM_6-22" select="'https://rdl.train.army.mil/catalog-ws/view/100.ATSC/3110F413-E915-47C1-85AA-DD4065C654C3-1274570636396/fm6_22.pdf'"/>
+   
+   <!--   p:file-copy not working on remote PDF  -->
+   <p:load href="{$FM_6-22}" message="[GRAB-FM6-22] Loading fm6_22.pdf ..."/>
+   <p:store href="lib/fm6_22.pdf" message="[GRAB-FM6-22] Saving fm6_22.pdf"/>
+      
+</p:declare-step>
\ No newline at end of file
diff --git a/projects/oscal-from-pdf/GRAB-RESOURCES.xpl b/projects/oscal-from-pdf/GRAB-RESOURCES.xpl
new file mode 100644
index 00000000..bb76b61d
--- /dev/null
+++ b/projects/oscal-from-pdf/GRAB-RESOURCES.xpl
@@ -0,0 +1,37 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<p:declare-step xmlns:p="http://www.w3.org/ns/xproc"
+   xmlns:c="http://www.w3.org/ns/xproc-step" version="3.0"
+   xmlns:ox="http://csrc.nist.gov/ns/oscal-xproc3"
+   type="ox:GRAB-RESOURCES"
+   name="GRAB-RESOURCES">
+   
+
+   <!-- Purpose: update local copies of some OSCAL resources from its release repository -->
+   
+   <!-- RNG for NISO STS is here: https://niso-sts.org/downloadables/RNGs/v1-2/NISO-STS-interchange-1-2-MathML3-RNG.zip
+   
+   See decision tree at https://niso-sts.org/DTDs-XSDs-RNGs-1-2.html
+   -->
+   <p:variable name="download-path" select="'https://github.com/usnistgov/OSCAL/releases/download/v1.1.2'"/>
+  
+   <!-- A $prefix is used to tag messages, expected to match the process type -->
+   <p:variable name="prefix" select="'[' || 'GRAB-RESOURCES' || ']'"/>
+   
+   <p:for-each message="{$prefix} Saving resources in ./lib ...">
+      <!-- iterating over each 'resource' as a discrete document node -->
+      <p:with-input>
+         <p:document href="{$download-path}/oscal_catalog_schema.xsd"/>
+         <!--<p:document href="{$download-path}/oscal_profile_schema.xsd"/>-->
+      </p:with-input>
+      <p:variable name="filename" select="p:document-property(.,'base-uri') ! replace(.,'.*/','')"/>
+      
+      <!-- No exception handling since a failed load often produces a result
+           making it difficult to anticipate what a failure looks like - -->
+      <!-- NB - A Schematron error comes back as soon as this is out of line with /*/@type i.e. the file name
+           - starting the @message with { $prefix } silences the error, but please reset $prefix -->
+      
+      <p:store message="{$prefix} ... saving { $filename }"
+               href="lib/{ $filename }"/>
+   </p:for-each>
+   
+</p:declare-step>
\ No newline at end of file
diff --git a/projects/oscal-from-pdf/extract-FM6_22-chapter4.xpl b/projects/oscal-from-pdf/extract-FM6_22-chapter4.xpl
index 99655a55..eb72ad7d 100644
--- a/projects/oscal-from-pdf/extract-FM6_22-chapter4.xpl
+++ b/projects/oscal-from-pdf/extract-FM6_22-chapter4.xpl
@@ -1,36 +1,94 @@
 <?xml version="1.0" encoding="UTF-8"?>
-<p:declare-step xmlns:p="http://www.w3.org/ns/xproc"
-   xmlns:c="http://www.w3.org/ns/xproc-step" version="3.0"
-   xmlns:ox="http://csrc.nist.gov/ns/oscal-xproc3"
-   xmlns:xsl="http://www.w3.org/1999/XSL/Transform"
+<p:declare-step xmlns:p="http://www.w3.org/ns/xproc" xmlns:c="http://www.w3.org/ns/xproc-step" version="3.0"
+   xmlns:ox="http://csrc.nist.gov/ns/oscal-xproc3" xmlns:xsl="http://www.w3.org/1999/XSL/Transform"
+   xmlns:xvrl="http://www.xproc.org/ns/xvrl"
    type="ox:MINIMAL"
-   name="MINIMAL"
-   xmlns="http://www.w3.org/1999/xhtml">
+   name="MINIMAL" xmlns="http://www.w3.org/1999/xhtml">
 
+
+   <!-- For samples and boilerplate see file ../../projects/xproc-doc/xproc-snippets.xml -->
    
-  <!-- For samples and boilerplate see file ../../projects/xproc-doc/xproc-snippets.xml -->
+   <!--<p:import href="src/xvrl-summarize.xpl"/>-->
 
-   <!--<p:output port="schematron-messages" serialization="map{'indent' : true(), 'omit-xml-declaration': true() }"/>-->
+   <p:option static="true" name="writing-all" select="false()"/>
    
-  <p:load href="source/export/fm6_22.html"/>
    
+   <p:output port="sts-validation-report" sequence="true"
+      serialization="map{'indent' : true(), 'omit-xml-declaration': true() }" pipe="summary@summarize-sts-validation"/>
+
+   <p:output port="oscal-validation-report" sequence="true"
+      serialization="map{'indent' : true(), 'omit-xml-declaration': true() }" pipe="summary@summarize-oscal-validation"/>
    
    
-  <!--<p:cast-content-type content-type="xml/application"/>-->
+   <!-- Ende Prolog -->
    
-   <!--<p:store href="temp/in_html" message="saving input" serialization="map{'indent' : true(), 'omit-xml-declaration': true() }"/>-->
+   <!-- A copy of the following step is also in file src/xvrl-summarize.xpl for import - 
+      here a local declaration is shown  -->
    
+   <!-- An XSLT is the other obvious way to do this, but XProc! --> 
+   <p:declare-step name="xvrl-summarize" type="ox:xvrl-summarize">
+      <!-- Expecting an xvrl:report or xvrl:NO_REPORT on the primary input port -->
+      <p:input port="xvrl-report" primary="true"/>
+      
+      <p:output port="summary" sequence="true"/>
+      
+      <p:variable name="here" select="resolve-uri('.') => p:urify()"/>
+      <p:variable name="schema" select="/xvrl:report/xvrl:metadata/xvrl:schema/p:urify(@href)"/>
+      <p:variable name="validation-errors" select="//xvrl:detection[@severity=('fatal-error','error')]"/>
+      <p:variable name="error-count" select="count($validation-errors)"/>
    
-   <!-- The HTML coming in is poorly and incorrectly structured, with contents crammed into list items
-     even across section boundaries
-     In particular, Chapter 4 starts inside a list item within a purported list that spans across chapters.
+      <p:choose>
+         <p:when test="empty(/xvrl:report)">
+            <p:identity>
+               <p:with-input port="source">
+                  <p:empty/>
+               </p:with-input>
+            </p:identity>
+         </p:when>
+         <p:when test="empty($validation-errors)">
+            <p:identity>
+               <p:with-input port="source">
+                  <p:inline xml:space="preserve">CONGRATULATIONS! No validation errors are reported against { substring-after($schema, $here) }</p:inline>
+               </p:with-input>
+            </p:identity>
+         </p:when>
+         <p:otherwise>
+            <p:identity>
+               <p:with-input port="source">
+                  <p:inline xml:space="preserve">Uhoh . . .        
+Validating result with { $schema } - { $error-count } {
+            if ($error-count eq 1) then 'error' else 'errors' } reported</p:inline>
+               </p:with-input>
+            </p:identity>
+         </p:otherwise>
+      </p:choose>
+   </p:declare-step>
    
-   A filter deals with this by flattening all lists and marking where list bullets are presumed to appear. -->
+   <!-- These schemas must be in place for validations to be performed -->
+   <!-- Break these to test pipeline behavior when they are missing  -->
+   <p:variable name="sts-rng" select="'lib/NISO-STS-interchange-1-MathML3-RNG/NISO-STS-interchange-1-mathml3.rng'"/>
+   <p:variable name="oscal-xsd" select="'lib/oscal_catalog_schema.xsd'"/>
+
+   <!-- for data serialization -->
+   <p:variable name="legible" select="map{'indent' : true(), 'omit-xml-declaration': true() }"/>
+
+   <!-- Anfangen  -->
+
+   <p:load href="source/export/fm6_22.html"/>
+
+   <!--
+      
+   The HTML coming in is poorly and incorrectly structured, with contents crammed into list items, even across section boundaries.
+       
+   In particular, Chapter 4 starts inside a list item within a purported list that spans across chapters.
    
+   A filter deals with this by flattening all lists and marking where list bullets are presumed to appear. -->
+
    <p:xslt>
       <p:with-input port="stylesheet">
          <p:inline expand-text="false">
-            <xsl:stylesheet version="3.0" xpath-default-namespace="http://www.w3.org/1999/xhtml" exclude-result-prefixes="#all">
+            <xsl:stylesheet version="3.0" xpath-default-namespace="http://www.w3.org/1999/xhtml"
+               exclude-result-prefixes="#all">
                <xsl:mode on-no-match="shallow-copy"/>
                <xsl:template match="ul">
                   <xsl:apply-templates/>
@@ -43,20 +101,11 @@
          </p:inline>
       </p:with-input>
    </p:xslt>
-   
-   <p:store href="temp/t01.html" message="Saving flattened input"  serialization="map{'indent' : true(), 'omit-xml-declaration': true() }"/>
-   
-   <!-- The path /*/body/p[normalize-space() => starts-with('Chapter')]
-   shows that Chapters appear cleanly demarcated by body/p[matches(normalize-space(.),'Chapter \d')]
-   so we will use these as our boundaries for pulling Chapter 4 only -->
 
-   
-   
-   <!--<p:delete match="/*/body/*[/*/body/p[normalize-space()='Chapter 4'] >> .] |
-      /*/body/*[. >> /*/body/p[normalize-space()='Chapter 5']] | /*/body/p[normalize-space()='Chapter 5']"/>-->
-   
-   <!-- Next we drop out everything but Chapter 4 - the above representing an effort to do this in
-        XProc-alone - couldn't get it to work and bothersomely slow -->
+   <p:store use-when="$writing-all" href="temp/t01.html" message="Saving flattened input"
+      serialization="map{'indent' : true(), 'omit-xml-declaration': true() }"/>
+
+   <!-- Next we drop out everything but Chapter 4 using a crude but effective method on this data set-->
    <p:xslt>
       <p:with-input port="stylesheet">
          <p:inline expand-text="false">
@@ -67,7 +116,7 @@
                <xsl:variable name="ch5" select="$ch5_head/(. | following-sibling::*)"/>
                <xsl:template match="body">
                   <xsl:copy>
-                    <xsl:apply-templates select="$ch4_head/(. | following-sibling::*) except $ch5"/>
+                     <xsl:apply-templates select="$ch4_head/(. | following-sibling::*) except $ch5"/>
                   </xsl:copy>
                </xsl:template>
             </xsl:stylesheet>
@@ -75,55 +124,126 @@
       </p:with-input>
    </p:xslt>
 
-   <p:variable name="legible" select="map{'indent' : true(), 'omit-xml-declaration': true() }"/>
-   
-   <p:store href="temp/t02.xml" message="Saving extracted chapter 4"  serialization="$legible"/>
    
+   <p:store use-when="$writing-all" href="temp/t02.xml" message="Saving extracted chapter 4"
+      serialization="$legible"/>
+
    <!-- Now we have content we can restructure - see the XSLTs for the logic -->
-   
+
    <p:xslt>
       <p:with-input port="stylesheet" href="src/fm22-6_structure.xsl"/>
    </p:xslt>
-   
+
    <!-- Now with nested structures, although some collect multiple items (indicated by paragraph numbers) and tables (to be joined) -->
    <p:xslt>
       <p:with-input port="stylesheet" href="src/fm22-6_restructure.xsl"/>
    </p:xslt>
-   
-   <p:store href="temp/t03_structured.xml" message="Saving chapter 4 re/structured"  serialization="$legible"/>
-   
+
+   <p:store use-when="$writing-all" href="temp/t03_structured.xml" message="Saving chapter 4 re/structured" serialization="$legible"/>
+
    <p:xslt>
       <p:with-input port="stylesheet" href="src/fm22-6-html-to-sts.xsl"/>
    </p:xslt>
-   
+
    <!-- This is valid STS, although not yet perfect -->
-   <p:store href="temp/t04_sts-rough.xml" message="STS conversion - simple mapping"  serialization="$legible"/>
-   
+   <p:store use-when="$writing-all" href="temp/t04_sts-rough.xml" message="STS conversion - simple mapping" serialization="$legible"/>
+
    <p:xslt>
       <p:with-input port="stylesheet" href="src/fm22-6_sts-enhance1.xsl"/>
    </p:xslt>
-   
+
    <!-- p is either labeled (enumerated) as in the source, or collected into lists -->
-   <p:store href="temp/t05_sts-corrected.xml" message="STS cleanup" serialization="$legible"/>
+   <p:store use-when="$writing-all" href="temp/t05_sts-corrected.xml" message="STS fixup" serialization="$legible"/>
+
+   <p:xslt>
+      <p:with-input port="stylesheet" href="src/fm22-6_sts-enhance2.xsl"/>
+   </p:xslt>
+
+   <p:store use-when="$writing-all" href="temp/t06_sts-enhanced.xml" message="STS cleanup" serialization="$legible"/>
+
+   <p:xslt>
+      <p:with-input port="stylesheet" href="src/fm22-6_sts-enhance3.xsl"/>
+   </p:xslt>
+
+   <!-- Saving before we validate, because validation produces a report as primary result-->
+   <p:store name="best-sts" message="Saving STS best-so-far - temp/FM_6-22-STS.xml" href="temp/FM_6-22-STS.xml" serialization="$legible"/>
+
+   <p:choose name="sts-validation">
+      <p:when test="doc-available($sts-rng)">
+         <p:output port="report" pipe="report@sts-rng-validation"/>
+         <p:validate-with-relax-ng assert-valid="false" name="sts-rng-validation"
+            message="Validating STS against RNG schema { $sts-rng }">
+            <p:with-input port="schema">
+               <p:document href="{ $sts-rng }"/>
+            </p:with-input>
+         </p:validate-with-relax-ng>
+      </p:when>
+      <p:otherwise>
+         <p:output port="report" pipe="result@no-rng"/>
+         <p:identity
+            message="Not validating STS - no schema found at { $sts-rng } - try running pipeline GRAB-NISO_STS-RNG.xpl"
+            name="no-rng">
+            <p:with-input port="source">
+               <p:inline>
+                  <NO_REPORT xmlns="http://www.xproc.org/ns/xvrl">Schema { $sts-rng } not found - try running pipeline
+                     GRAB-NISO_STS-RNG.xpl</NO_REPORT>
+               </p:inline>
+            </p:with-input>
+         </p:identity>
+      </p:otherwise>
+   </p:choose>
+
+   <ox:xvrl-summarize name="summarize-sts-validation"/>
+
+   <!-- Now we have STS and have checked it, we can produce OSCAL -->
+
+   <p:xslt>
+      <p:with-input port="source" pipe="result@best-sts"/>
+      <p:with-input port="stylesheet" href="src/fm22-6_sts-to-oscal.xsl"/>
+   </p:xslt>
+
+   <!-- Some whitespace cleanup after XSLT shuffling -->
+   <p:string-replace xmlns:o="http://csrc.nist.gov/ns/oscal/1.0"
+      match="o:p/text()[1][empty(preceding-sibling::*)]" replace="replace(.,'^\s+','')"/>
    
+   <p:store message="Saving OSCAL - temp/FM_6-22-OSCAL.xml" href="temp/FM_6-22-OSCAL.xml" serialization="$legible"/>
+
+   <p:choose name="oscal-validation">
+      <p:when test="doc-available($oscal-xsd)">
+         <p:output port="report" pipe="report@oscal-xsd-validation"/>
+         <p:validate-with-xml-schema assert-valid="false" name="oscal-xsd-validation"
+            message="Validating OSCAL against XSD { $oscal-xsd }">
+            <p:with-input port="schema">
+               <p:document href="{ $oscal-xsd }"/>
+            </p:with-input>
+         </p:validate-with-xml-schema>
+      </p:when>
+      <p:otherwise>
+         <p:output port="report" pipe="result@no-xsd"/>
+         <p:identity message="Not validating OSCAL - no schema found at { $oscal-xsd }" name="no-xsd">
+            <p:with-input port="source">
+               <p:inline>
+                  <NO_REPORT xmlns="http://www.xproc.org/ns/xvrl">Schema { $oscal-xsd } not found - try running pipeline
+                     GRAB-RESOURCES.xpl</NO_REPORT>
+               </p:inline>
+            </p:with-input>
+         </p:identity>
+      </p:otherwise>
+   </p:choose>
+
+   <ox:xvrl-summarize name="summarize-oscal-validation"/>
+
+   <!-- Next, OSCAL -
+   capability tables into OSCAL controls
+     'Evaluation' includes Strength and Need Indicators and Underlying Causes
+     'Capability Growth' includees Feedback, Study and Practice parts
    
+   main text?
    
-   <!-- further corrections and enhancements -
-          consolidate paragraphs with no target into targeted paragraphs (group-starts-with?)
-          cross-references? at least to tables (T|t)able\s+4\-\d\d?
-          mark up <abbrev> around acronyms replacing "Legend" lines
-          map 'legend' tr into table-footer
-          break out disp-quote in "Employing Leadership Requirements Model Development Activities" box
-        
-        validate STS for consistency?
-          src/fm22-6_chapter4.sch
-          tables
-        
-        convert to OSCAL
+   -->
 
--->
 
-<!-- Analytic pass: for section, build a table containing cells with each of:
+   <!-- Analytic pass: for section, build a table containing cells with each of:
    strength_indicators (signs of strength)
    need_indicators (lapses / shortfalls)
    underlying_causes (shadow forms, negative projections or inversions)
@@ -136,22 +256,9 @@
         li mismanagement
         split tables
         flat structures / anomalous formatting
-      
-      structure nested sections via implicit headers on p/@class
-      validate these against tables pulled from Table 4-4 and Table 4-5
         
-      The plan is to start with the tables for each 
-      These are the regularities we expect from these tables:
-   We should see tables 4-6 through 4-80 in total (75 tables)
-   tr 1 leads with "Strength Indicators" in td[1] and "Need Indicators" in td[2]
-   tr 2 has two cells
-   tr 3 spans two columns, reading "underlying causes"
-   tr 4 / count(td) = 1
-   tr 5 / td[1] reads 'Feedback'
-   row 5 / td[2] reads 'Study'
-   row 5 / td[3] reads 'Practice'
-   each of tr[5] has count(td)=2-->
-   
+-->
+
    <!--<p:validate-with-schematron assert-valid="true" name="table-check"
       message="Checking chapter 4 table integrity in data capture ...">
       <p:with-input port="schema" href="src/fm22-6_chapter4.sch"/>
@@ -160,5 +267,5 @@
    <p:identity name="schematron-messages">
       <p:with-input port="source" pipe="report@table-check"/>
    </p:identity>-->
-   
+
 </p:declare-step>
\ No newline at end of file
diff --git a/projects/oscal-from-pdf/readme.md b/projects/oscal-from-pdf/readme.md
index 6f92b6ec..fb242b0f 100644
--- a/projects/oscal-from-pdf/readme.md
+++ b/projects/oscal-from-pdf/readme.md
@@ -69,44 +69,46 @@ A sequence of transformations reproduces the data set with NISO STS (Standards T
   - Repairing broken callout boxes
   - Consolidating partial/broken tables into tables
   - Grouping bulleted items into lists
+  - Making spot corrections
 
-Etc. Details can be traced in the [XProc pipeline file extract-FM6_22-chapter4.xpl](extract-FM6_22-chapter4.xpl).
+The last category includes a number of tactical interventions to correct or enhance encoding where the text has been wrongly or inadequately tagged at earlier stages - note that these changes all serve to *reveal* not *hide* the original, and in every case the XSLT documents unambiguously any changes being made.
 
-The result is Chapter 4 *only* in a correct and harmonious STS encoding.
+Details can be traced in the [XProc pipeline file extract-FM6_22-chapter4.xpl](extract-FM6_22-chapter4.xpl).
 
-As currently configured (still in development) this pipeline writes results - intermediate as well as final - to the [temp](temp) directory.
+The result is Chapter 4 *only* in a correct and harmonious STS encoding.
 
-In particular, the file [temp/t05_sts-corrected.xml](temp/t05_sts-corrected.xml) in that directory (produced by the pipeline) is valid NIST STS XML.
+As currently configured (still in development) this pipeline writes results - intermediate as well as final - to the [temp](temp) directory. Using a 'diff' tool on any consecutive pair of files here shows the changes made by the transformation that produces the later from the earlier.
 
 View this file using NISO STS Tools such as the [NISO STS Viewer](https://pages.nist.gov/xslt-blender/sts-viewer/).
 
 NISO STS makes a good intermediate model for this enhancement, as it
-
-- Presents a comprehensive, retrospective encoding of the document as received, stabilizing a semantic representation, without entanglement in the next task, namely mapping this (or any) semantically adequate representation into OSCAL
+- Presents a comprehensive, retrospective encoding of the document as received, stabilizing a semantic representation, without entanglement in the related but different problem of mapping this (or any) semantically adequate representation into OSCAL
 - Can be inspected, tested and validated on the way through, including with bespoke validation, STS display tools and other methods
-- Produces a useful spin-off artifact (the STS instance itself)
+- Produces a useful spin-off artifact: the STS instance itself
 
 With a clean STS representation of the document in hand, casting into OSCAL will be straightforward.
 
 The main advantages of this stepwise process are in transparencey and traceability, duplicability, and debuggability.
 
+#### NISO STS quality check
 
-#### NISO STS to OSCAL
+If it is adequate in its semantic description, we should be able to validate the h*ck out of the data in its STS form. (This is circular reasoning: if it is invalid, we call it inadequate.)
 
-TODO: this is where we pause ...
+The constraint set we use to assert this validation is in three tiers:
 
-OSCAL will be free-form text (with links) followed by a control sequence
+1. NISO STS validation - the pipeline [GRAB-NISO_STS-RNG.xpl](GRAB-NISO_STS-RNG.xpl) acquires a copy of an RNG schema for this purpose
+1. Schematron validation - house rules (NIST RLM) - see [src/sts-check.sch](src/sts-check.sch)
+1. Schematron validation - bespoke rules asserting regularities for this instance - see [src/fm22-6_chapter4.sch](src/fm22-6_chapter4.sch)
+   Note: this Schematron is used to report errors and anomalies in sources or intermediate STS files that are then remediated in subsequent phases
 
-The XProc to perform this conversion is *TBD*
+#### NISO STS to OSCAL
+
+- Tables 4-6 and on (the 'competency tables') become controls with parts
+- The narrative sequence is dropped into its own free-flowing OSCAL (nested parts)
 
-keep section sequence replacing tables with links to control structures - competency/attribute/component
-          
-drill all the way to part[@class='item'] with line items
-        create control structures for tables 4-6 - 4-80
-        
-Retain tables 1-5 as plain-old tables
-          break numbered sections out into parts
-          separate out a formal part for capabilities / indicators
+TODO: Schematron and check internal cross-referencing
+
+OSCAL will be free-form text (with links) followed by a control sequence
 
 ### Initial planning and survey
 
diff --git a/projects/oscal-from-pdf/src/fm22-6-html-to-sts.xsl b/projects/oscal-from-pdf/src/fm22-6-html-to-sts.xsl
index de8cc2f8..04144b5f 100644
--- a/projects/oscal-from-pdf/src/fm22-6-html-to-sts.xsl
+++ b/projects/oscal-from-pdf/src/fm22-6-html-to-sts.xsl
@@ -84,7 +84,7 @@
    <xsl:template match="p">
       <p>
          <xsl:for-each select="preceding-sibling::*[1]/self::bullet">
-            <xsl:attribute name="style-type">bulleted</xsl:attribute>
+            <xsl:attribute name="style-type">bullet</xsl:attribute>
          </xsl:for-each>
          <xsl:apply-templates/>
       </p>
diff --git a/projects/oscal-from-pdf/src/fm22-6_chapter4-sts.sch b/projects/oscal-from-pdf/src/fm22-6_chapter4-sts.sch
new file mode 100644
index 00000000..9db85232
--- /dev/null
+++ b/projects/oscal-from-pdf/src/fm22-6_chapter4-sts.sch
@@ -0,0 +1,48 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<sch:schema xmlns:sch="http://purl.oclc.org/dsdl/schematron" queryBinding="xslt2"
+   xmlns:sqf="http://www.schematron-quickfix.com/validator/process">
+   
+   
+   <sch:pattern>
+      <sch:rule context="p">
+         <sch:let name="exception" value="matches(.,'[…:\?\.][”\)]?\s*$') or exists(ancestor::table-wrap )"/>
+         <sch:assert test="child::*[1] is child::target[1] or $exception">Punctuation check.</sch:assert>
+      </sch:rule>
+      <sch:rule context="table-wrap">
+         <sch:let name="table-label" value="label"/>
+         <sch:let name="other-tables" value="//table-wrap except ."/>
+         <sch:assert test="empty($other-tables[label=$table-label])">Table label '<sch:value-of select="label"/>' is not distinctive</sch:assert>
+         
+      </sch:rule>
+      <sch:rule context="table-wrap[@custom-type=('competency','attribute')]/table/tbody">
+         <sch:assert test="tr[1]/td[1]!normalize-space(.)='Strength Indicators'">table out of order (Strength Indicators label missing)</sch:assert>
+         <sch:assert test="tr[1]/td[2]!normalize-space(.)='Need Indicators'">table out of order ('Need Indicators' label missing)</sch:assert>
+         <sch:assert test="(tr[2]/td => count()) eq 2">table out of order</sch:assert>
+         <sch:assert test="tr[3]/td[1]!normalize-space(.)='Underlying Causes'">table out of order ('Underlying Causes' label missing)</sch:assert>
+         <sch:assert test="(tr[4]/td => count()) eq 1">table out of order</sch:assert>
+         <sch:assert test="tr[5]/td[1]!normalize-space(.)='Feedback'">table out of order ('Feedback' label missing)</sch:assert>
+         <sch:assert test="tr[6]/td[1]!normalize-space(.)='Study'">table out of order ('Study' label missing)</sch:assert>
+         <sch:assert test="tr[7]/td[1]!normalize-space(.)='Practice'">table out of order ('Practice' label missing)</sch:assert>
+      </sch:rule>
+         <!--These are the regularities we expect from these tables:
+      
+      tr 1 leads with "Strength Indicators" in td[1] and "Need Indicators" in td[2]
+      tr 2 has two cells
+      tr 3 spans two columns, reading "underlying causes"
+      tr 4 / count(td) = 1
+      tr 5 / td[1] reads 'Feedback'
+      row 5 / td[2] reads 'Study'
+      row 5 / td[3] reads 'Practice'
+      each of tr[5] has count(td)=2
+      -->
+   </sch:pattern>
+   
+   <sch:pattern>
+      <sch:rule context="body/p | sec/p">
+         <sch:let name="exception" value="starts-with(.,'Tip:')"/>
+         <sch:assert test="child::*[1] is child::target[1] or $exception">We expect p element to start with a target.</sch:assert>
+      </sch:rule>
+      
+   </sch:pattern>
+   
+</sch:schema>
\ No newline at end of file
diff --git a/projects/oscal-from-pdf/src/fm22-6_chapter4.sch b/projects/oscal-from-pdf/src/fm22-6_chapter4.sch
deleted file mode 100644
index 73820cd9..00000000
--- a/projects/oscal-from-pdf/src/fm22-6_chapter4.sch
+++ /dev/null
@@ -1,13 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<sch:schema xmlns:sch="http://purl.oclc.org/dsdl/schematron" queryBinding="xslt2"
-   xmlns:sqf="http://www.schematron-quickfix.com/validator/process">
-   
-   
-   <sch:pattern>
-     <sch:rule context="body/p | sec/p">
-        <sch:assert test="child::*[1] is child::target[1]">We expect p element to start with a target.</sch:assert>
-     </sch:rule>
-   
-   </sch:pattern>
-   
-</sch:schema>
\ No newline at end of file
diff --git a/projects/oscal-from-pdf/src/fm22-6_sts-enhance1.xsl b/projects/oscal-from-pdf/src/fm22-6_sts-enhance1.xsl
new file mode 100644
index 00000000..b8e9739d
--- /dev/null
+++ b/projects/oscal-from-pdf/src/fm22-6_sts-enhance1.xsl
@@ -0,0 +1,183 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<xsl:stylesheet version="3.0"
+   xmlns:xsl="http://www.w3.org/1999/XSL/Transform"
+   xmlns:xs="http://www.w3.org/2001/XMLSchema"
+   xmlns:xlink="http://www.w3.org/1999/xlink"
+   exclude-result-prefixes="#all">
+
+   <!--<xsl:output indent="true"/>-->
+
+   <xsl:mode on-no-match="shallow-copy"/>
+
+<!--splitting secs into:
+everything split apart except 
+table-wrap grouped by @id
+p when bulleted
+drop p when not bulleted?
+pull in boxed-text/p per sec
+repair boxed-text where needed
+
+-->
+
+<xsl:template match="/*">
+   <xsl:processing-instruction name="xml-model">href="../lib/NISO-STS-interchange-1-MathML3-RNG/NISO-STS-interchange-1-mathml3.rng" type="application/xml" schematypens="http://relaxng.org/ns/structure/1.0"</xsl:processing-instruction>
+   <xsl:processing-instruction name="xml-model">href="../src/fm22-6_chapter4.sch" type="application/xml" schematypens="http://purl.oclc.org/dsdl/schematron"</xsl:processing-instruction>
+   <xsl:next-match/>
+</xsl:template>
+   
+<!-- Repairing the botch job produced by Acrobat and its HTML export here -->
+   <xsl:template
+      match="boxed-text[string(.) => normalize-space() => starts-with('Determining Developmental Activities')]">
+      <boxed-text>
+         <caption>
+            <title>Determining Developmental Activities</title>
+         </caption>
+         <p>Answer these to select appropriate developmental activities:</p>
+         <list list-type="simple">
+            <list-item>
+               <p><bold>Developmental Activity:</bold> How do I need to improve?</p>
+            </list-item>
+            <list-item>
+               <p><bold>Desired Outcome:</bold> What do I hope to achieve? </p>
+            </list-item>
+            <list-item>
+               <p><bold>Method:</bold> How am I going to do this? What resources do I need? </p>
+            </list-item>
+            <list-item>
+               <p><bold>Time available:</bold> When will I do this? How will I monitor progress (such as identifying and monitoring milestones, rewarding success, or identifying accountability partners)?</p>
+            </list-item>
+            <list-item>
+               <p><bold>Limits:</bold> What factors will affect or hinder successfully implementing this activity? </p>
+            </list-item>
+            <list-item>
+               <p><bold>Controls:</bold> What minimizes or controls the factors that hinder implementing this activity?</p>
+            </list-item>
+         </list>
+      </boxed-text>
+   </xsl:template>
+   
+   <xsl:template match="p[. = 'Employing Leadership Requirements Model Developmental Activities']">
+      <boxed-text>
+         <caption>
+         <title>
+            <xsl:apply-templates/>
+         </title>
+         </caption>
+         <!-- including the following p elements through to the sec end -->
+         <xsl:copy-of select="following-sibling::*"/>
+      </boxed-text>
+   </xsl:template>
+      
+   <xsl:template match="p[preceding-sibling::p = 'Employing Leadership Requirements Model Developmental Activities']"/>
+   
+   
+  <xsl:template match="sec">
+     <xsl:copy>
+        <xsl:copy-of select="@*"/>
+     <xsl:for-each-group select="*" group-adjacent="(self::p/@style-type,self::table-wrap/@id,'fallback')[1]">
+        <xsl:choose>
+           <xsl:when test="current-grouping-key() = 'fallback'">
+              <xsl:apply-templates select="current-group()"/>
+           </xsl:when>
+           <xsl:when test="current-grouping-key() = 'bullet'">
+              <list list-type="bullet">
+                 <xsl:for-each select="current-group()">
+                    <list-item>
+                       <xsl:apply-templates select="."/>
+                    </list-item>
+                 </xsl:for-each>
+              </list>
+           </xsl:when>
+           <xsl:when test="starts-with(current-grouping-key(),'table')">
+              <table-wrap id="{ current-grouping-key() }">
+                 <xsl:apply-templates select="./label"/>
+                 <xsl:apply-templates select="./caption"/>
+                 <table>
+                    <tbody>
+                       <xsl:apply-templates select="current-group()//tr"/>
+                    </tbody>
+                 </table>
+              </table-wrap>
+           </xsl:when>
+           <xsl:otherwise>
+              <xsl:apply-templates select="current-group()"/>
+              <xsl:message>Unexpected unnaccounted-for element(s) being grouped on key { current-grouping-key() }</xsl:message>
+           </xsl:otherwise>
+        </xsl:choose>
+<!-- groups - current-grouping-key() is '' spill out
+           except special case
+        @bulleted these are p to be grouped as list
+        table ID value - table to be grouped
+        
+        -->
+     </xsl:for-each-group>
+     </xsl:copy>
+  </xsl:template>
+   
+   <xsl:template match="body/p[matches(.,'^Table \d\-\d\d?')] | sec/p[matches(.,'^Table \d\-\d\d?')]"/>
+   
+   <xsl:template match="p/@style-type"/>
+   
+   <xsl:variable name="pid-regex" as="xs:string">^\s*4\-\d\d?\d?\.</xsl:variable>
+   
+   <xsl:template match="p/text()[1][matches(.,$pid-regex)]">
+      <xsl:analyze-string select="string(.)" regex="{$pid-regex}">
+         <xsl:matching-substring>
+            <target id="p{ normalize-space(.) ! replace(.,'\.$','') }">
+               <xsl:value-of select="normalize-space(.)"/>
+            </target>
+         </xsl:matching-substring>
+         <xsl:non-matching-substring>
+            <xsl:value-of select="."/>
+         </xsl:non-matching-substring>
+      </xsl:analyze-string>
+   </xsl:template>
+  
+  <!--Further adjustments:-->
+    <!--Table 4-31 has a cell incorrectly labeled "Needs Indicators"-->
+   
+   <xsl:template match="tbody/tr/td/p[.='Needs Indicators']">
+      <p>Need Indicators</p>
+   </xsl:template>
+  
+   <!--Table 4-78 has a spurious table row capturing text that belongs in the last paragraph of the preceding table row.-->
+   
+   <xsl:template match="tbody/tr[normalize-space(.)='and prioritize needs. Identify candidates for immediate feedback and coaching.']"/>
+   
+   <xsl:template match="tbody/tr[following-sibling::tr[1]/normalize-space(.)='and prioritize needs. Identify candidates for immediate feedback and coaching.']/td[2]/p[last()]">
+      
+      <xsl:copy>
+         <xsl:apply-templates/>
+         <xsl:text> </xsl:text>
+         <xsl:for-each select="parent::td/../following-sibling::tr[1]/td[1]/p">
+            <xsl:apply-templates/>
+         </xsl:for-each>
+      </xsl:copy>
+      
+   </xsl:template>
+   
+   
+<!--
+      
+Remediations  to data produced from the HTML sources -
+
+- Repairing boxed-text
+  x "Determining Developmental Activities" in section 4-7
+  x "Employing Leadership Requirements Model Developmental Activities" in 4-9
+   x Grouping list items into lists
+   x Consolidating tables where now broken
+   x Pull enumerations from p
+
+To validate results:
+- diff ../temp/t04_sts-rough.xml and ../temp/t05_sts-nicer.xml
+- tables : all in order, all regular, structured correctly?
+    (next step being to cast into OSCAL)
+- paragraphs - numbered properly?
+- compare a generated vs the received copy of Table 4-4
+
+-->
+
+
+
+
+</xsl:stylesheet>
\ No newline at end of file
diff --git a/projects/oscal-from-pdf/src/fm22-6_sts-enhance2.xsl b/projects/oscal-from-pdf/src/fm22-6_sts-enhance2.xsl
new file mode 100644
index 00000000..18f60241
--- /dev/null
+++ b/projects/oscal-from-pdf/src/fm22-6_sts-enhance2.xsl
@@ -0,0 +1,143 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<xsl:stylesheet version="3.0"
+   xmlns:xsl="http://www.w3.org/1999/XSL/Transform"
+   xmlns:xs="http://www.w3.org/2001/XMLSchema"
+   xmlns:xlink="http://www.w3.org/1999/xlink"
+   exclude-result-prefixes="#all">
+
+   <!--<xsl:output indent="true"/>-->
+
+   <xsl:mode on-no-match="shallow-copy"/>
+
+<!-- Remediating broken paragraphs emitted by Adobe Acrobat in HTML production. -->
+   
+   <xsl:template match="sec | list-item | th | td">
+      <xsl:copy>
+         <xsl:copy-of select="@*"/>
+         <xsl:for-each-group select="child::*" group-adjacent="exists(self::p)">
+           <xsl:choose>
+              <xsl:when test="current-grouping-key()">
+                 <xsl:call-template name="break-paragraphs"/>
+              </xsl:when>
+              <xsl:otherwise>
+                 <xsl:apply-templates select="current-group()"/>
+              </xsl:otherwise>
+           </xsl:choose>
+         </xsl:for-each-group>
+      </xsl:copy>
+   </xsl:template>
+   
+   <xsl:template name="break-paragraphs">
+     <xsl:for-each-group select="current-group()" group-ending-with="*[matches(.,'[\?\.]\)?\s*$')]">
+        <p>
+           <xsl:for-each select="current-group()">
+              <xsl:if test="position() > 1">
+                 <xsl:text> </xsl:text>
+              </xsl:if>
+              <xsl:apply-templates select="child::node()"/>
+           </xsl:for-each>
+        </p>
+     </xsl:for-each-group>
+      
+   </xsl:template>
+   
+   
+   <!-- No longer need this marker -->
+   <xsl:template match="named-content">
+      <xsl:apply-templates/>
+   </xsl:template>
+   
+   <xsl:template match="p/text()[following-sibling::*[1]/self::named-content]">
+      <italic>
+         <xsl:copy-of select="normalize-space(.)"/>
+      </italic>
+      <xsl:text> </xsl:text>
+   </xsl:template>
+   
+   <xsl:template match="italic[ends-with(.,' ')]">
+      <xsl:copy>
+      <xsl:value-of select="normalize-space(.)"/>
+      </xsl:copy>
+      <xsl:text> </xsl:text>
+   </xsl:template>
+   
+   <xsl:template match="boxed-text[caption/title='Employing Leadership Requirements Model Developmental Activities']">
+      <xsl:copy>
+         <xsl:copy-of select="@*"/>
+         <xsl:apply-templates mode="dropping-quote"/>
+         <disp-quote>
+            <xsl:apply-templates mode="keeping-quote"/>
+         </disp-quote>
+      </xsl:copy>
+   </xsl:template>
+   
+   <xsl:mode name="dropping-quote" on-no-match="deep-copy"/>
+   
+   <xsl:template match="p[starts-with(.,'~from')]" mode="dropping-quote"/>
+   <xsl:template match="p[starts-with(following-sibling::*[1]/self::p,'~from')]" mode="dropping-quote"/>
+   
+   <xsl:template match="*" mode="keeping-quote"/>
+   
+   <xsl:template match="p[starts-with(.,'~from')]" mode="keeping-quote">
+      <attrib>
+         <xsl:apply-templates/>
+      </attrib>
+   </xsl:template>
+   
+   <xsl:template match="p[starts-with(following-sibling::*[1]/self::p,'~from')]" mode="keeping-quote">
+      <xsl:copy-of select="."/>
+   </xsl:template>
+   
+   <xsl:template match="table-wrap[@id=preceding-sibling::table-wrap/@id]" priority="28"/>
+   
+   <!-- A list of keywords that *happen* to mark competencies by virtue of appearing in table 4-5 -->
+   <xsl:variable name="competencies" select="'leads', 'builds','extends','communicates','prepares','creates','develops','stewards','gets'"/>
+   
+   <xsl:template match="table-wrap[table/descendant::td = 'Strength Indicators']">
+      <xsl:copy>
+         <xsl:copy-of select="@*"/>
+         <!-- If one of the 'competency' keywords is found among titles of sections containing this table, it's a 'competency' control -->
+         <xsl:attribute name="custom-type" expand-text="true">{ if (ancestor::sec/title/lower-case(.)!tokenize(.,'\s+') = $competencies) then 'competency' else 'attribute' }</xsl:attribute>
+         <xsl:apply-templates/>
+         <xsl:apply-templates select="descendant::tr[starts-with(normalize-space(.),'Legend:')]" mode="table-glossary"/>
+      </xsl:copy>
+   </xsl:template>
+   
+   <xsl:template match="tr" mode="table-glossary">
+      <xsl:variable name="marked-row">
+         <xsl:analyze-string select="normalize-space(.)" regex="[A-Z][A-Z]+">
+            <xsl:matching-substring>
+               <term>
+                  <xsl:value-of select="."/>
+               </term>
+            </xsl:matching-substring>
+            <xsl:non-matching-substring>
+               <xsl:value-of select="."/>
+            </xsl:non-matching-substring>
+         </xsl:analyze-string>
+      </xsl:variable>
+      <def-list>
+         <title>Legend</title>
+         <xsl:iterate select="$marked-row/term" expand-text="true">
+            <def-item>
+               <xsl:copy-of select="."/>
+               <def>
+                  <p>{ following-sibling::text()[1] ! normalize-space(.) }</p>
+               </def>
+            </def-item>
+         </xsl:iterate>
+      </def-list>
+   </xsl:template>
+   
+   <!-- Dropping these since the values for the Legend are pulled into the def-list above -->
+   <xsl:template match="tr[starts-with(normalize-space(.),'Legend:')]"/>
+  
+   <xsl:template match="tbody">
+      <xsl:variable name="tableID" select="parent::table/parent::table-wrap/@id"/>
+      <xsl:copy>
+         <xsl:copy-of select="@*"/>
+         <xsl:apply-templates select="parent::table/parent::table-wrap/../child::table-wrap[@id=$tableID]/table/tbody/*"/>
+      </xsl:copy>
+   </xsl:template>
+   
+</xsl:stylesheet>
\ No newline at end of file
diff --git a/projects/oscal-from-pdf/src/fm22-6_sts-enhance3.xsl b/projects/oscal-from-pdf/src/fm22-6_sts-enhance3.xsl
new file mode 100644
index 00000000..65647c8a
--- /dev/null
+++ b/projects/oscal-from-pdf/src/fm22-6_sts-enhance3.xsl
@@ -0,0 +1,39 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<xsl:stylesheet version="3.0"
+   xmlns:xsl="http://www.w3.org/1999/XSL/Transform"
+   xmlns:xs="http://www.w3.org/2001/XMLSchema"
+   xmlns:xlink="http://www.w3.org/1999/xlink"
+   exclude-result-prefixes="#all">
+
+   <!--<xsl:output indent="true"/>-->
+
+   <xsl:mode on-no-match="shallow-copy"/>
+
+   <xsl:template match="p[target/@id='p4-8']/text()" priority="2">
+      <xsl:analyze-string select="string(.)" regex="tables 4\-4 and 4\-5">
+         <xsl:matching-substring>
+            <xsl:text>tables </xsl:text>
+            <xref rid="table4_4" ref-type="table">4-4</xref>
+            <xsl:text> and </xsl:text>
+            <xref rid="table4_5" ref-type="table">4-5</xref>
+         </xsl:matching-substring>
+         <xsl:non-matching-substring>
+            <xsl:value-of select="."/>
+         </xsl:non-matching-substring>
+      </xsl:analyze-string>
+   </xsl:template>
+   
+   <xsl:template match="p/text()">
+      <xsl:analyze-string select="string(.)" regex="[tT]able 4\-(\d\d?)">
+         <xsl:matching-substring>
+            <xref rid="table4_{ regex-group(1) }" ref-type="table">
+               <xsl:value-of select="normalize-space(.)"/>
+            </xref>
+         </xsl:matching-substring>
+         <xsl:non-matching-substring>
+            <xsl:value-of select="."/>
+         </xsl:non-matching-substring>
+      </xsl:analyze-string>
+   </xsl:template>
+   
+</xsl:stylesheet>
\ No newline at end of file
diff --git a/projects/oscal-from-pdf/src/fm22-6_sts-to-oscal.xsl b/projects/oscal-from-pdf/src/fm22-6_sts-to-oscal.xsl
new file mode 100644
index 00000000..c15114e9
--- /dev/null
+++ b/projects/oscal-from-pdf/src/fm22-6_sts-to-oscal.xsl
@@ -0,0 +1,252 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<xsl:stylesheet version="3.0"
+   xmlns:xsl="http://www.w3.org/1999/XSL/Transform"
+   xmlns:xs="http://www.w3.org/2001/XMLSchema"
+   xmlns:xlink="http://www.w3.org/1999/xlink"
+   xmlns:ox="http://csrc.nist.gov/ns/oscal-xproc3"
+   exclude-result-prefixes="#all"
+   xmlns="http://csrc.nist.gov/ns/oscal/1.0">
+
+   <xsl:output indent="true"/>
+
+   <xsl:mode on-no-match="text-only-copy"/>
+
+   <xsl:template match="/processing-instruction()"/>
+   
+   <xsl:template match="/*">
+      
+      <catalog uuid="27d2b146-2a4b-4a4e-9562-f18564b77b5c">
+         <metadata>
+            <title>Electronic Transcription - US Army Field Manual 6-22: Developing Leaders (November 2022) - Chapter 4: Learning and Developmental Activities</title>
+            <last-modified>2024-09-10T15:19:38.6995852-04:00</last-modified>
+            <version>0.8 draft</version>
+            <oscal-version>1.1.2</oscal-version>
+            <remarks>
+               <p>This encoded representation of Field Manual 6-22 was produced from published source <em>without explicit authorization from</em> or <em>any coordination with</em> the document's originators or the US Army. Reliance on this representation without reference to the publication from which it is derived is not advised.</p>
+               <p>See the repository readme for further documentation.</p>
+            </remarks>
+         </metadata>
+         <group id="full-text">
+            <title>Learning and Developmental Activities</title>
+            <xsl:apply-templates/>
+         </group>
+         <group id="attributes">
+            <title>Attributes</title>
+            <xsl:apply-templates select="body/descendant::table-wrap[@custom-type = 'attribute']"
+               mode="competency-controls"/>
+         </group>
+         <group id="competencies">
+            <title>Competencies</title>
+            <xsl:apply-templates select="body/descendant::table-wrap[@custom-type = 'competency']"
+               mode="competency-controls"/>
+         </group>
+      </catalog>
+   </xsl:template>
+
+
+   <xsl:template match="front"/>
+   
+   <xsl:template match="body">
+      <part name="intro">
+         <xsl:apply-templates select="* except sec"/>
+      </part>
+      <xsl:apply-templates select="sec"/>
+      
+   </xsl:template>
+   
+   <xsl:template match="tbody | caption">
+      <xsl:apply-templates/>
+   </xsl:template>
+   
+   <xsl:template match="caption/title">
+      <title>
+         <xsl:apply-templates/>
+      </title>
+   </xsl:template>
+   
+   <xsl:template match="sec">
+      <!-- for the @id, mapping space to _ and stripping characters not permitted in XML names -->
+      <control id="{ title/normalize-space() => lower-case() => translate(' /','__') => replace('\C','') }">
+         <xsl:apply-templates select="title"/>
+         <xsl:for-each-group select="* except (title|sec)" group-starting-with="p[exists(target)]">
+            <xsl:choose>
+               <xsl:when test="exists(target)">
+                  <part name="enumeration" id="part-{  replace(target,'\.$','') }">
+                     <xsl:for-each select="child::target">
+                        <prop name="label" value="{ normalize-space(.) }"/>
+                     </xsl:for-each>
+                     <xsl:apply-templates select="current-group()"/>
+                  </part>
+               </xsl:when>
+            </xsl:choose>
+         </xsl:for-each-group>
+         <xsl:apply-templates select="sec"/>
+      </control>
+   </xsl:template>
+   
+   <xsl:template match="target"/>
+
+   <xsl:variable name="lookup-tables" select="//table-wrap[@id=('table4_4','table4_5')]/table"/>
+
+   <xsl:function name="ox:category-for-control" as="xs:string?">
+      <xsl:param name="tableID" as="xs:string"/>
+      <xsl:variable name="linking-row" select="$lookup-tables/tbody/tr[td/p/xref/@rid=$tableID]"/>
+      <xsl:sequence select="($linking-row | $linking-row/preceding-sibling::tr)[count(td) eq 4][last()]/td[1]/normalize-space(.)"/>
+   </xsl:function>
+   
+   <xsl:function name="ox:subcategory-for-control" as="xs:string?">
+      <xsl:param name="tableID" as="xs:string"/>
+      <xsl:variable name="linking-row"     select="$lookup-tables/tbody/tr[td/p/xref/@rid=$tableID]"/>
+      <xsl:sequence select="($linking-row | $linking-row/preceding-sibling::tr)[count(td) = (3,4)][last()]/td[2]/normalize-space(.)"/>
+   </xsl:function>
+   
+   <xsl:template match="p[starts-with(.,'Tip:')]">
+      <part name="tip">
+         <prop name="label" value="Tip:"/>
+         <p>
+           <xsl:apply-templates/>
+         </p>
+      </part>
+   </xsl:template>
+   
+   <xsl:template match="p[starts-with(.,'Tip:')]/text()[1]">
+      <xsl:text expand-text="true">{ substring-after(.,'Tip: ') }</xsl:text>
+   </xsl:template>
+      
+   <xsl:template match="table-wrap">
+      <part id="{@id}" name="directory">
+         <xsl:apply-templates select="caption, label"/>
+         <xsl:apply-templates select="* except (caption | label)"/>         
+      </part>
+   </xsl:template>
+   
+   <xsl:template match="boxed-text">
+      <part name="boxed">
+         <xsl:apply-templates/>
+      </part>
+   </xsl:template>
+   
+   <xsl:template match="disp-quote">
+      <part name="quoted">
+         <xsl:apply-templates/>
+      </part>
+   </xsl:template>
+   
+   <xsl:template match="sec/title">
+      <title>
+         <xsl:apply-templates/>
+      </title>
+   </xsl:template>
+   
+   <xsl:template match="table | tr | th | td">
+      <xsl:element name="{ name() }" namespace="http://csrc.nist.gov/ns/oscal/1.0">
+         <xsl:apply-templates/>
+      </xsl:element>
+   </xsl:template>
+   
+   <xsl:template match="label">
+      <prop name="label" value="{normalize-space(.)}"/>
+   </xsl:template>
+
+   <!-- Dropping in default (no mode) traversal -->
+   <xsl:template match="table-wrap[@custom-type=('competency','attribute')]">
+      <link href="#{@id}" rel="related"/>
+   </xsl:template>
+   
+   <xsl:key name="by-id" match="*[exists(@id)]" use="@id"/>
+   
+   <xsl:template match="table-wrap" mode="competency-controls" expand-text="true">
+      <xsl:variable name="class" select="if (@id = key('by-id','table4_4')//td/p/xref/@rid) then 'attribute' else 'competency'"/>
+      <control id="{ @id }" class="{ $class }">
+         <title>{ caption/title/normalize-space() }</title>
+         <prop name="category"    value="{ ox:category-for-control(@id/string(.)) }"/>
+         <prop name="subcategory" value="{ ox:subcategory-for-control(@id) }"/>
+         
+         <part name="evaluation">
+            <xsl:apply-templates mode="#current" select="table/tbody/tr[2]/td[1]">
+               <xsl:with-param name="title">Strength Indicators</xsl:with-param>
+            </xsl:apply-templates>
+            <xsl:apply-templates mode="#current" select="table/tbody/tr[2]/td[2]">
+               <xsl:with-param name="title">Need Indicators</xsl:with-param>
+            </xsl:apply-templates>
+            <xsl:apply-templates mode="#current" select="table/tbody/tr[4]/td[1]">
+               <xsl:with-param name="title">Underlying Causes</xsl:with-param>
+            </xsl:apply-templates>
+         </part>
+         <part name="growth-activities">
+            <xsl:apply-templates mode="#current" select="table/tbody/tr[5]/td[2]">
+               <xsl:with-param name="title">Feedback</xsl:with-param>
+            </xsl:apply-templates>
+            <xsl:apply-templates mode="#current" select="table/tbody/tr[6]/td[2]">
+               <xsl:with-param name="title">Study</xsl:with-param>
+            </xsl:apply-templates>
+            <xsl:apply-templates mode="#current" select="table/tbody/tr[7]/td[2]">
+               <xsl:with-param name="title">Practice</xsl:with-param>
+            </xsl:apply-templates>
+         </part>
+      </control>
+   </xsl:template>
+   
+   <xsl:template mode="competency-controls" match="td" expand-text="true">
+      <xsl:param name="title">Untitled</xsl:param>
+      <part name="{ $title ! lower-case(.) ! translate(.,' ','_') }">
+         <title>{ $title }</title>
+         <xsl:apply-templates/>
+      </part>
+   </xsl:template>
+   
+   <xsl:template match="list">
+      <ul>
+         <xsl:apply-templates/>
+      </ul>
+   </xsl:template>
+   
+   <xsl:template match="list-item">
+      <li>
+         <xsl:apply-templates/>
+      </li>
+   </xsl:template>
+   
+   <xsl:template match="table-wrap[empty(@custom-type)]//td/p">
+      <xsl:for-each select="preceding-sibling::*[1]"><br/></xsl:for-each>
+      <xsl:apply-templates/>
+   </xsl:template>
+   
+   <xsl:template match="p | attrib">
+      <p>
+         <xsl:apply-templates/>
+      </p>
+   </xsl:template>
+   
+   <xsl:template match="bold">
+      <b>
+         <xsl:apply-templates/>
+      </b>
+   </xsl:template>
+   
+   <xsl:template match="em">
+      <emph>
+         <xsl:apply-templates/>
+      </emph>
+   </xsl:template>
+   
+   <xsl:template match="italic">
+      <i>
+         <xsl:apply-templates/>
+      </i>
+   </xsl:template>
+   
+   <xsl:template match="xref">
+      <a href="#{@rid}">
+         <xsl:apply-templates/>
+      </a>
+   </xsl:template>
+   
+   <xsl:template match="*">
+      <xsl:message expand-text="true">Not matching { name() }</xsl:message>
+      <xsl:copy copy-namespaces="no">
+         <xsl:apply-templates/>
+      </xsl:copy>
+   </xsl:template>
+   
+</xsl:stylesheet>
\ No newline at end of file
diff --git a/projects/oscal-from-pdf/src/oscal-check.sch b/projects/oscal-from-pdf/src/oscal-check.sch
new file mode 100644
index 00000000..332509c8
--- /dev/null
+++ b/projects/oscal-from-pdf/src/oscal-check.sch
@@ -0,0 +1,18 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<sch:schema xmlns:sch="http://purl.oclc.org/dsdl/schematron" queryBinding="xslt2"
+    xmlns:sqf="http://www.schematron-quickfix.com/validator/process"
+    xmlns:xsl="http://www.w3.org/1999/XSL/Transform"
+    xmlns:o="http://csrc.nist.gov/ns/oscal/1.0">
+    
+    <sch:ns prefix="o"     uri="http://csrc.nist.gov/ns/oscal/1.0"/>
+    <sch:ns prefix="oscal" uri="http://csrc.nist.gov/ns/oscal/1.0"/>
+    
+    <xsl:key name="xref" match="*[exists(@id)]" use="'#' || @id"/>
+   
+   <sch:pattern>
+      <sch:rule context="oscal:a[starts-with(@href,'#')]">
+         <sch:assert test="key('xref',@href) => exists()">Internal link href='<sch:value-of select="@href"/>' has no target</sch:assert>
+      </sch:rule>
+   </sch:pattern>
+   
+</sch:schema>
\ No newline at end of file
diff --git a/projects/oscal-from-pdf/src/xvrl-summarize.xpl b/projects/oscal-from-pdf/src/xvrl-summarize.xpl
new file mode 100644
index 00000000..5c58bfc7
--- /dev/null
+++ b/projects/oscal-from-pdf/src/xvrl-summarize.xpl
@@ -0,0 +1,50 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<p:declare-step
+   xmlns:p="http://www.w3.org/ns/xproc"
+   xmlns:c="http://www.w3.org/ns/xproc-step" version="3.0"
+   xmlns:ox="http://csrc.nist.gov/ns/oscal-xproc3"
+   xmlns:xsl="http://www.w3.org/1999/XSL/Transform"
+   xmlns:xvrl="http://www.xproc.org/ns/xvrl"
+   name="xvrl-summarize" type="ox:xvrl-summarize"
+   >
+   
+   <!-- Expecting an xvrl:report or xvrl:NO_REPORT on the primary input port 
+        SEE ../extract-FM6_22-chapter4.xpl-->
+   <p:input port="xvrl-report"/>
+   
+   
+   <p:output port="summary" sequence="true"/>
+
+   <p:variable name="here" select="resolve-uri('..') => p:urify()"/>
+   <p:variable name="schema" select="/xvrl:report/xvrl:metadata/xvrl:schema/p:urify(@href)"/>
+   <p:variable name="validation-errors" select="//xvrl:detection[@severity=('fatal-error','error')]"/>
+   <p:variable name="error-count" select="count($validation-errors)"/>
+   
+   
+   <p:choose>
+      <p:when test="exists(/xvrl:NO_REPORT)">
+         <p:identity>
+            <p:with-input port="source">
+               <p:empty/>
+            </p:with-input>
+         </p:identity>
+      </p:when>
+      <p:when test="empty($validation-errors)">
+         <p:identity>
+            <p:with-input port="source">
+               <p:inline xml:space="preserve">CONGRATULATIONS! No validation errors are reported against { substring-after($schema, $here) }</p:inline>
+            </p:with-input>
+         </p:identity>
+      </p:when>
+      <p:otherwise>
+         <p:identity>
+            <p:with-input port="source">
+               <p:inline xml:space="preserve">Uhoh . . .        
+Validating result with { $schema } - { $error-count } {
+            if ($error-count eq 1) then 'error' else 'errors' } reported</p:inline>
+            </p:with-input>
+         </p:identity>
+      </p:otherwise>
+   </p:choose>
+      
+</p:declare-step>
\ No newline at end of file