Merge pull request #2 from metanorma/rt-iso-10303-issue-65

feat: complete migration from metanorma/iso-10303#65, rubocop fixes a…

lib/suma/collection_config.rb

@@ -18,9 +18,5 @@ def self.from_file(path)
     def to_file(path)
       File.open(path, "w") { |f| f.write to_yaml }
     end
-
-    def expand_schemas_only(path)
-      manifest.expand_schemas_only(path)
-    end
   end
 end

lib/suma/collection_manifest.rb

@@ -8,6 +8,8 @@ module Suma
   class CollectionManifest < Metanorma::Collection::Config::Manifest
     attribute :schemas_only, Shale::Type::Boolean
     attribute :entry, CollectionManifest, collection: true
+    # attribute :schema_source, Shale::Type::String
+    attr_accessor :schema_config
 
     yaml do
       map "identifier", to: :identifier
@@ -32,23 +34,84 @@ def docref_from_yaml(model, value)
       model.entry = CollectionManifest.from_yaml(value.to_yaml)
     end
 
-    def expand_schemas_only(path)
-      if schemas_only
-        doc = YAML.safe_load(File.read(file))
-        schemas = YAML.safe_load(File.read(File.join(File.dirname(file), "schemas.yaml")))
-        self.file = nil
-        self.title = "ISO Collection"
-        self.type = "collection"
-        entries = schemas["schemas"].each_with_object([]) do |(k, v), m|
-          fname = Pathname(v["path"]).each_filename.to_a
-          fname[-1].sub!(/exp$/, "xml") # we compile these in col.compile below
-          m << CollectionManifest.new(identifier: k, title: k, file: File.join(path, fname[-2], "doc_#{fname[-1]}"))
+    def export_schema_config(path)
+      export_config = @schema_config || Suma::SchemaConfig::Config.new
+      return export_config unless entry
+
+      entry.each do |x|
+        child_config = x.export_schema_config(path)
+        export_config.concat(child_config) if child_config
+      end
+
+      export_config
+    end
+
+    def lookup(attr_sym, match)
+      results = entry.select { |e| e.send(attr_sym) == match }
+      results << self if send(attr_sym) == match
+      results
+    end
+
+    def process_entry(schema_output_path)
+      return [self] unless entry
+
+      ret = entry.each_with_object([]) do |e, m|
+        add = e.expand_schemas_only(schema_output_path)
+        m.concat(add)
+      end
+
+      self.entry = ret
+      [self]
+    end
+
+    def expand_schemas_only(schema_output_path)
+      return process_entry(schema_output_path) unless file
+
+      # If there is collection.yml, this is a document collection, we process
+      # schemas.yaml.
+      if File.basename(file) == 'collection.yml'
+        schemas_yaml_path = File.join(File.dirname(file), "schemas.yaml")
+        if schemas_yaml_path && File.exist?(schemas_yaml_path)
+          @schema_config = Suma::SchemaConfig::Config.from_file(schemas_yaml_path)
         end
-        doc = Array(CollectionManifest.new(title: doc["bibdata"]["docid"]["id"], type: "document", entry: entries))
-        self.entry = doc
-      else
-        entry&.each { |e| e.expand_schemas_only(path) }
       end
+
+      return process_entry(schema_output_path) unless schemas_only
+
+      # If we are going to keep the schemas-only file and compile it, we can't
+      # have it showing up in output.
+      self.index = false
+
+      doc = CollectionConfig.from_file(file)
+      doc_id = doc.bibdata.id
+
+      entries = @schema_config.schemas.map do |schema|
+        fname = [File.basename(schema.path, ".exp"), ".xml"].join
+
+        CollectionManifest.new(
+          identifier: schema.id,
+          title: schema.id,
+          file: File.join(schema_output_path, "doc_#{schema.id}", fname),
+          # schema_source: schema.path
+        )
+      end
+
+      # we need to separate this file from the following new entries
+      added = CollectionManifest.new(
+        title: "Collection",
+        type:  "collection",
+        identifier: self.identifier + "_"
+      )
+
+      added.entry = [
+        CollectionManifest.new(
+          title: doc_id,
+          type: "document",
+          entry: entries,
+        ),
+      ]
+
+      [self, added]
     end
   end
 end

lib/suma/express_schema.rb

@@ -8,12 +8,9 @@ module Suma
   class ExpressSchema
     attr_accessor :path, :id, :parsed, :output_path
 
-    def initialize(path:, output_path:)
+    def initialize(id:, path:, output_path:)
       @path = Pathname.new(path).expand_path
-      @parsed = Expressir::Express::Parser.from_file(@path.to_s)
-      Utils.log "Loaded EXPRESS schema: #{path}"
-
-      @id = @parsed.schemas.first.id
+      @id = id
       @output_path = output_path
     end
 
@@ -28,8 +25,17 @@ def type
       end
     end
 
+    def parsed
+      return @parsed if @parsed
+
+      @parsed = Expressir::Express::Parser.from_file(@path.to_s)
+      Utils.log "Loaded EXPRESS schema: #{path}"
+      @id = @parsed.schemas.first.id
+      @parsed
+    end
+
     def to_plain
-      @parsed.to_s(no_remarks: true)
+      parsed.to_s(no_remarks: true)
     end
 
     def filename_plain

lib/suma/processor.rb

@@ -12,22 +12,6 @@
 module Suma
   class Processor
     class << self
-      # Can move to schema_config.rb
-      def write_all_schemas(schemas_all_path, document_paths)
-        all_schemas = Suma::SchemaConfig::Config.new(path: schemas_all_path)
-
-        document_paths.each do |path|
-          schemas_yaml = File.join(File.dirname(path), "schemas.yaml")
-          next unless File.exist?(schemas_yaml)
-
-          schemas_config = Suma::SchemaConfig::Config.from_file(schemas_yaml)
-          all_schemas.concat(schemas_config)
-        end
-
-        Utils.log "Writing #{schemas_all_path}..."
-        all_schemas.to_file
-        Utils.log "Done."
-      end
 
       def run(metanorma_yaml_path:, schemas_all_path:, compile:, output_directory: "_site")
         Utils.log "Current directory: #{Dir.getwd}"
@@ -39,16 +23,20 @@ def run(metanorma_yaml_path:, schemas_all_path:, compile:, output_directory: "_s
         collection_config_path = site_config.metanorma.source.files.first
         collection_config = Suma::CollectionConfig.from_file(collection_config_path)
         collection_config.path = collection_config_path
+        collection_config.manifest.expand_schemas_only("plain_schemas")
 
-        # Gather all the inner (per-document) collection.yml files
-        document_paths = collection_config.manifest.entry.map(&:file)
+        exported_schema_config = collection_config.manifest.export_schema_config(schemas_all_path)
+        exported_schema_config.path = schemas_all_path
 
-        write_all_schemas(schemas_all_path, document_paths)
+        Utils.log "Writing #{schemas_all_path}..."
+        exported_schema_config.to_file
+        Utils.log "Done."
 
         col = Suma::SchemaCollection.new(
           config_yaml: schemas_all_path,
+          manifest: collection_config.manifest,
           output_path_docs: "schema_docs",
-          output_path_schemas: "plain_schemas"
+          output_path_schemas: "plain_schemas",
         )
 
         if compile
@@ -97,16 +85,17 @@ def run(metanorma_yaml_path:, schemas_all_path:, compile:, output_directory: "_s
         if compile
           Utils.log "Compiling complete collection..."
 
+          # TODO: Why will defining a collection immediately compile??
           metanorma_collection = Metanorma::Collection.parse(collection_config_path)
 
           # TODO: Somehow this is no longer used
           collection_opts = {
             format: [:html],
             output_folder: output_directory,
             compile: {
-              no_install_fonts: true
+              no_install_fonts: true,
             },
-            coverpage: "cover.html"
+            coverpage: "cover.html",
           }
           metanorma_collection.render(collection_opts)
 

lib/suma/schema_attachment.rb

@@ -6,7 +6,7 @@
 
 module Suma
   class SchemaAttachment
-    attr_accessor :schema, :output_path, :config
+    attr_accessor :schema, :output_path, :config, :id
 
     def initialize(schema:, output_path:)
       @schema = schema
@@ -100,13 +100,13 @@ def compile
       save_adoc
 
       relative_path = Pathname.new(filename_adoc).relative_path_from(Dir.pwd)
-      Utils.log "Compiling schema #{relative_path}..."
+      Utils.log "Compiling schema (id: #{id}, type: #{self.class}) => #{relative_path}"
       Metanorma::Compile.new.compile(
         filename_adoc,
         agree_to_terms: true,
         no_install_fonts: true
       )
-      Utils.log "Compiling schema #{filename_adoc}...done!"
+      Utils.log "Compiling schema (id: #{id}, type: #{self.class}) => #{relative_path}... done!"
 
       # clean_artifacts
 

lib/suma/schema_collection.rb

@@ -8,11 +8,12 @@
 
 module Suma
   class SchemaCollection
-    attr_accessor :config, :schemas, :docs, :output_path_docs, :output_path_schemas
+    attr_accessor :config, :schemas, :docs, :output_path_docs, :output_path_schemas,
+      :manifest
 
-    def initialize(config: nil, config_yaml: nil, output_path_docs: nil, output_path_schemas: nil, document_paths: nil)
-      @schemas = []
-      @docs = []
+    def initialize(config: nil, config_yaml: nil, output_path_docs: nil, output_path_schemas: nil, manifest: nil)
+      @schemas = {}
+      @docs = {}
       @schema_name_to_docs = {}
       @output_path_docs = if output_path_docs
                             Pathname.new(output_path_docs).expand_path
@@ -31,38 +32,55 @@ def initialize(config: nil, config_yaml: nil, output_path_docs: nil, output_path
                   SchemaConfig::Config.from_file(config_yaml)
                 end
 
-      return self unless @config
+      @manifest = manifest
+    end
+
+    def doc_from_schema_name(schema_name)
+      @schema_name_to_docs[schema_name]
+    end
 
-      @config.schemas.each do |config_schema|
+    def process_schemas(schemas, klass)
+      schemas.each do |config_schema|
         s = ExpressSchema.new(
-          path: config_schema.path,
-          output_path: @output_path_schemas
+          id: config_schema.id,
+          path: config_schema.path.to_s,
+          output_path: @output_path_schemas.to_s
+        )
+
+        doc = klass.new(
+          schema: s,
+          output_path: @output_path_docs.join(s.id)
         )
-        doc = if config_schema.schemas_only
-                SchemaDocument.new(
-                  schema: s,
-                  output_path: @output_path_docs.join(s.id)
-                )
-              else
-                SchemaAttachment.new(
-                  schema: s,
-                  output_path: @output_path_docs.join(s.id)
-                )
-              end
-        @docs << doc
-        @schemas << s
+
+        @docs[s.id] = doc
+        @schemas[s.id] = s
         @schema_name_to_docs[s.id] = doc
       end
     end
 
-    def doc_from_schema_name(schema_name)
-      @schema_name_to_docs[schema_name]
+    def finalize
+      # Process each schema in @config.schemas
+      process_schemas(@config.schemas, SchemaAttachment)
+
+      manifest_entry = @manifest.lookup(:schemas_only, true)
+
+      manifest_entry.each do |entry|
+        next unless entry.schema_config
+
+        # Process each schema in entry.schema_config.schemas
+        process_schemas(entry.schema_config.schemas, SchemaDocument)
+      end
     end
 
     def compile
-      schemas.map(&:save_exp)
+      finalize
+      schemas.each_pair do |schema_id, entry|
+        entry.save_exp
+      end
 
-      docs.each(&:compile)
+      docs.each_pair do |schema_id, entry|
+        entry.compile
+      end
 
       # TODO: make this parallel
       # Utils.log"Starting Ractor processing"