Merge pull request #10924 from obophenotype/refactor-build-system

Refactor build pipeline and add synonyms to some translations

.github/workflows/qc.yml

@@ -19,7 +19,7 @@ jobs:
   ontology_qc:
     # The type of runner that the job will run on
     runs-on: ubuntu-latest
-    container: obolibrary/odkfull:v1.5.2
+    container: obolibrary/odkfull:v1.5.4
 
     # Steps represent a sequence of tasks that will be executed as part of the job
     steps:

src/ontology/Makefile

@@ -10,7 +10,7 @@
 # More information: https://github.com/INCATools/ontology-development-kit/
 
 # Fingerprint of the configuration file when this Makefile was last generated
-CONFIG_HASH=                0fcd636feea85fce01a9e5009487aad06aedf4ba32833cb10c92bcf68fbc2cc2
+CONFIG_HASH=                3a5afcd02a25dc79d50036cccc5351438f736858c55ac01f7abd367aa2dd6172
 
 
 # ----------------------------------------
@@ -63,7 +63,7 @@ PATTERN_RELEASE_FILES=      $(PATTERNDIR)/definitions.owl $(PATTERNDIR)/pattern.
 
 TRANSLATIONSDIR=            ../translations
 BABELONPY=                  babelon -q
-TRANSLATIONS_OWL=$(TRANSLATIONSDIR)/hp-de.babelon.owl $(TRANSLATIONSDIR)/hp-it.babelon.owl $(TRANSLATIONSDIR)/hp-nl.babelon.owl  $(TRANSLATIONSDIR)/hp-nl.synonyms.owl $(TRANSLATIONSDIR)/hp-fr.babelon.owl  $(TRANSLATIONSDIR)/hp-fr.synonyms.owl $(TRANSLATIONSDIR)/hp-cs.babelon.owl  $(TRANSLATIONSDIR)/hp-cs.synonyms.owl $(TRANSLATIONSDIR)/hp-pt.babelon.owl  $(TRANSLATIONSDIR)/hp-pt.synonyms.owl $(TRANSLATIONSDIR)/hp-tr.babelon.owl $(TRANSLATIONSDIR)/hp-zh.babelon.owl $(TRANSLATIONSDIR)/hp-nna.babelon.owl $(TRANSLATIONSDIR)/hp-tw.babelon.owl $(TRANSLATIONSDIR)/hp-dtp.babelon.owl $(TRANSLATIONSDIR)/hp-ja.babelon.owl $(TRANSLATIONSDIR)/hp-es.babelon.owl 
+TRANSLATIONS_OWL=$(TRANSLATIONSDIR)/hp-de.babelon.owl  $(TRANSLATIONSDIR)/hp-de.synonyms.owl $(TRANSLATIONSDIR)/hp-it.babelon.owl  $(TRANSLATIONSDIR)/hp-it.synonyms.owl $(TRANSLATIONSDIR)/hp-nl.babelon.owl  $(TRANSLATIONSDIR)/hp-nl.synonyms.owl $(TRANSLATIONSDIR)/hp-fr.babelon.owl  $(TRANSLATIONSDIR)/hp-fr.synonyms.owl $(TRANSLATIONSDIR)/hp-cs.babelon.owl  $(TRANSLATIONSDIR)/hp-cs.synonyms.owl $(TRANSLATIONSDIR)/hp-pt.babelon.owl  $(TRANSLATIONSDIR)/hp-pt.synonyms.owl $(TRANSLATIONSDIR)/hp-tr.babelon.owl  $(TRANSLATIONSDIR)/hp-tr.synonyms.owl $(TRANSLATIONSDIR)/hp-zh.babelon.owl  $(TRANSLATIONSDIR)/hp-zh.synonyms.owl $(TRANSLATIONSDIR)/hp-nna.babelon.owl $(TRANSLATIONSDIR)/hp-tw.babelon.owl $(TRANSLATIONSDIR)/hp-dtp.babelon.owl $(TRANSLATIONSDIR)/hp-ja.babelon.owl  $(TRANSLATIONSDIR)/hp-ja.synonyms.owl $(TRANSLATIONSDIR)/hp-es.babelon.owl  $(TRANSLATIONSDIR)/hp-es.synonyms.owl 
 TRANSLATIONS_TSV=$(TRANSLATIONSDIR)/hp-de-preprocessed.babelon.tsv $(TRANSLATIONSDIR)/hp-it-preprocessed.babelon.tsv $(TRANSLATIONSDIR)/hp-nl-preprocessed.babelon.tsv $(TRANSLATIONSDIR)/hp-fr-preprocessed.babelon.tsv $(TRANSLATIONSDIR)/hp-cs-preprocessed.babelon.tsv $(TRANSLATIONSDIR)/hp-pt-preprocessed.babelon.tsv $(TRANSLATIONSDIR)/hp-tr-preprocessed.babelon.tsv $(TRANSLATIONSDIR)/hp-zh-preprocessed.babelon.tsv $(TRANSLATIONSDIR)/hp-nna-preprocessed.babelon.tsv $(TRANSLATIONSDIR)/hp-tw-preprocessed.babelon.tsv $(TRANSLATIONSDIR)/hp-dtp-preprocessed.babelon.tsv $(TRANSLATIONSDIR)/hp-ja-preprocessed.babelon.tsv $(TRANSLATIONSDIR)/hp-es-preprocessed.babelon.tsv 
 TRANSLATION_FILES=$(TRANSLATIONSDIR)/$(ONT)-all.babelon.tsv $(TRANSLATIONSDIR)/$(ONT)-all.babelon.json
 
@@ -690,6 +690,9 @@ TRANSLATE_PREDICATES=rdfs:label
 $(TRANSLATIONSDIR)/hp-de.babelon.tsv:
 	wget "https://raw.githubusercontent.com/obophenotype/hpo-translations/main/babelon/hp-de.babelon.tsv" -O $@
 
+$(TRANSLATIONSDIR)/hp-de.synonyms.tsv:
+	wget "https://raw.githubusercontent.com/obophenotype/hpo-translations/main/babelon/hp-de.synonyms.tsv" -O $@
+
 
 
 $(TRANSLATIONSDIR)/hp-de-preprocessed.babelon.tsv: $(TRANSLATIONS_ONTOLOGY) $(TRANSLATIONSDIR)/hp-de.babelon.tsv
@@ -709,6 +712,9 @@ $(TRANSLATIONSDIR)/hp-de-preprocessed.babelon.tsv: $(TRANSLATIONS_ONTOLOGY) $(TR
 $(TRANSLATIONSDIR)/hp-it.babelon.tsv:
 	wget "https://raw.githubusercontent.com/obophenotype/hpo-translations/main/babelon/hp-it.babelon.tsv" -O $@
 
+$(TRANSLATIONSDIR)/hp-it.synonyms.tsv:
+	wget "https://raw.githubusercontent.com/obophenotype/hpo-translations/main/babelon/hp-it.synonyms.tsv" -O $@
+
 
 
 $(TRANSLATIONSDIR)/hp-it-preprocessed.babelon.tsv: $(TRANSLATIONS_ONTOLOGY) $(TRANSLATIONSDIR)/hp-it.babelon.tsv
@@ -816,6 +822,9 @@ $(TRANSLATIONSDIR)/hp-pt-preprocessed.babelon.tsv: $(TRANSLATIONS_ONTOLOGY) $(TR
 $(TRANSLATIONSDIR)/hp-tr.babelon.tsv:
 	wget "https://raw.githubusercontent.com/obophenotype/hpo-translations/main/babelon/hp-tr.babelon.tsv" -O $@
 
+$(TRANSLATIONSDIR)/hp-tr.synonyms.tsv:
+	wget "https://raw.githubusercontent.com/obophenotype/hpo-translations/main/babelon/hp-tr.synonyms.tsv" -O $@
+
 
 
 $(TRANSLATIONSDIR)/hp-tr-preprocessed.babelon.tsv: $(TRANSLATIONS_ONTOLOGY) $(TRANSLATIONSDIR)/hp-tr.babelon.tsv
@@ -835,6 +844,9 @@ $(TRANSLATIONSDIR)/hp-tr-preprocessed.babelon.tsv: $(TRANSLATIONS_ONTOLOGY) $(TR
 $(TRANSLATIONSDIR)/hp-zh.babelon.tsv:
 	wget "https://raw.githubusercontent.com/obophenotype/hpo-translations/main/babelon/hp-zh.babelon.tsv" -O $@
 
+$(TRANSLATIONSDIR)/hp-zh.synonyms.tsv:
+	wget "https://raw.githubusercontent.com/obophenotype/hpo-translations/main/babelon/hp-zh.synonyms.tsv" -O $@
+
 
 
 $(TRANSLATIONSDIR)/hp-zh-preprocessed.babelon.tsv: $(TRANSLATIONS_ONTOLOGY) $(TRANSLATIONSDIR)/hp-zh.babelon.tsv
@@ -909,7 +921,10 @@ $(TRANSLATIONSDIR)/hp-dtp-preprocessed.babelon.tsv: $(TRANSLATIONS_ONTOLOGY) $(T
 
 
 $(TRANSLATIONSDIR)/hp-ja.babelon.tsv:
-	wget "https://raw.githubusercontent.com/ogishima/HPO-Japanese/master/HPO-japanese.alpha.21Jul2023.tsv" -O $@
+	wget "https://raw.githubusercontent.com/obophenotype/hpo-translations/main/babelon/hp-ja.babelon.tsv" -O $@
+
+$(TRANSLATIONSDIR)/hp-ja.synonyms.tsv:
+	wget "https://raw.githubusercontent.com/obophenotype/hpo-translations/main/babelon/hp-ja.synonyms.tsv" -O $@
 
 
 
@@ -930,6 +945,9 @@ $(TRANSLATIONSDIR)/hp-ja-preprocessed.babelon.tsv: $(TRANSLATIONS_ONTOLOGY) $(TR
 $(TRANSLATIONSDIR)/hp-es.babelon.tsv:
 	wget "https://raw.githubusercontent.com/obophenotype/hpo-translations/main/babelon/hp-es.babelon.tsv" -O $@
 
+$(TRANSLATIONSDIR)/hp-es.synonyms.tsv:
+	wget "https://raw.githubusercontent.com/obophenotype/hpo-translations/main/babelon/hp-es.synonyms.tsv" -O $@
+
 
 
 $(TRANSLATIONSDIR)/hp-es-preprocessed.babelon.tsv: $(TRANSLATIONS_ONTOLOGY) $(TRANSLATIONSDIR)/hp-es.babelon.tsv

src/ontology/build-without-imports.sh

@@ -8,7 +8,6 @@ echo docker pull obolibrary/odkfull:$ODK
 
 ODK_TAG=$ODK sh run.sh make hpoa_clean -B
 test -f tmp/hpo-annotation-data/README.md
-ODK_TAG=$ODK sh run.sh make MIR=false IMP=false prepare_release -B
 ODK_TAG=$ODK sh run.sh make MIR=false IMP=false babelon prepare_release -B
 ODK_TAG=$ODK sh run.sh make hpoa -B
 ODK_TAG=$ODK sh run.sh make hpo_diff -B

src/ontology/hp-odk.yaml

@@ -109,13 +109,17 @@ babelon_translation_group:
   products:
     - id: hp-de
       auto_translate: FALSE
+      include_robot_template_synonyms: TRUE
       maintenance: mirror
+      mirror_synonyms_from: https://raw.githubusercontent.com/obophenotype/hpo-translations/main/babelon/hp-de.synonyms.tsv
       mirror_babelon_from: https://raw.githubusercontent.com/obophenotype/hpo-translations/main/babelon/hp-de.babelon.tsv
       language: de
     - id: hp-it
       auto_translate: FALSE
+      include_robot_template_synonyms: TRUE
       maintenance: mirror
       mirror_babelon_from: https://raw.githubusercontent.com/obophenotype/hpo-translations/main/babelon/hp-it.babelon.tsv
+      mirror_synonyms_from: https://raw.githubusercontent.com/obophenotype/hpo-translations/main/babelon/hp-it.synonyms.tsv
       language: it
     - id: hp-nl
       include_robot_template_synonyms: TRUE
@@ -142,13 +146,17 @@ babelon_translation_group:
       mirror_babelon_from: https://raw.githubusercontent.com/obophenotype/hpo-translations/main/babelon/hp-pt.babelon.tsv
       mirror_synonyms_from: https://raw.githubusercontent.com/obophenotype/hpo-translations/main/babelon/hp-pt.synonyms.tsv
     - id: hp-tr
+      include_robot_template_synonyms: TRUE
       language: tr
       maintenance: mirror
       mirror_babelon_from: https://raw.githubusercontent.com/obophenotype/hpo-translations/main/babelon/hp-tr.babelon.tsv
+      mirror_synonyms_from: https://raw.githubusercontent.com/obophenotype/hpo-translations/main/babelon/hp-tr.synonyms.tsv
     - id: hp-zh
+      include_robot_template_synonyms: TRUE
       language: zh
       maintenance: mirror
       mirror_babelon_from: https://raw.githubusercontent.com/obophenotype/hpo-translations/main/babelon/hp-zh.babelon.tsv
+      mirror_synonyms_from: https://raw.githubusercontent.com/obophenotype/hpo-translations/main/babelon/hp-zh.synonyms.tsv
     - id: hp-nna
       language: nna
       maintenance: mirror
@@ -162,13 +170,17 @@ babelon_translation_group:
       maintenance: mirror
       mirror_babelon_from: https://raw.githubusercontent.com/obophenotype/hpo-translations/main/babelon/hp-dtp.babelon.tsv
     - id: hp-ja
+      include_robot_template_synonyms: TRUE
       language: ja
       maintenance: mirror
-      mirror_babelon_from: https://raw.githubusercontent.com/ogishima/HPO-Japanese/master/HPO-japanese.alpha.21Jul2023.tsv
+      mirror_babelon_from: https://raw.githubusercontent.com/obophenotype/hpo-translations/main/babelon/hp-ja.babelon.tsv
+      mirror_synonyms_from: https://raw.githubusercontent.com/obophenotype/hpo-translations/main/babelon/hp-ja.synonyms.tsv
     - id: hp-es
+      include_robot_template_synonyms: TRUE
       language: es
       maintenance: mirror
       mirror_babelon_from: https://raw.githubusercontent.com/obophenotype/hpo-translations/main/babelon/hp-es.babelon.tsv
+      mirror_synonyms_from: https://raw.githubusercontent.com/obophenotype/hpo-translations/main/babelon/hp-es.synonyms.tsv
 robot_report:
   release_reports: True
   fail_on : ERROR

src/ontology/hp.Makefile

@@ -637,3 +637,5 @@ $(TMPDIR)/hp-%-merged.owl: hp-base.owl tmp/%.owl
 mappings: 
 	$(MAKE_FAST) ../mappings/hp-snomed.lexmatch.sssom.tsv
 
+babelon:
+	pip install -U babelon==0.3.4 --break-system-packages

src/translations/hp-es.synonyms.tsv

@@ -0,0 +1,2 @@
+subject_id	translation_value	comment
+ID	AL oboInOwl:hasExactSynonym@es

src/translations/hp-it.synonyms.tsv

@@ -0,0 +1,2 @@
+subject_id	translation_value	comment
+ID	AL oboInOwl:hasExactSynonym@it

src/translations/hp-tr.synonyms.tsv

@@ -0,0 +1,2 @@
+subject_id	translation_value	comment
+ID	AL oboInOwl:hasExactSynonym@tr