diff --git a/.gitignore b/.gitignore index d7e9a0b..0095e5a 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,3 @@ -*.conllu *.pyc *.ipynb diff --git a/requirements.txt b/requirements.txt index 98ea625..c70f2c9 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,6 +3,7 @@ build>=1.2.1 flake8>=7.1.0 jsonargparse[signatures]>=4.32.0 lightning>=2.4.0 +parameterized>=0.9.0 pytest>=7.4.4 transformers>=4.44.0 torch>=2.4.0 diff --git a/tests/testdata/datasets/README.md b/tests/testdata/datasets/README.md new file mode 100644 index 0000000..5c1a228 --- /dev/null +++ b/tests/testdata/datasets/README.md @@ -0,0 +1,5 @@ +This directory contains small "toy" samples drawn from Universal Dependencies +data for English, Greek, and Russian for `udtube_test.py`. The `_train.conllu` +files are used to train and validate the model, and the `_expected.conllu` +files are the result of applying the model to the training data. +Each file contains ten sentences. diff --git a/tests/testdata/datasets/el_expected.conllu b/tests/testdata/datasets/el_expected.conllu new file mode 100644 index 0000000..0dc2f13 --- /dev/null +++ b/tests/testdata/datasets/el_expected.conllu @@ -0,0 +1,270 @@ +# newdoc id = gdt-20120309-elwikinews-5160 +# sent_id = gdt-20120309-elwikinews-5160-1 +# text = Η Μάντσεστερ Γιουνάιτεντ ηττήθηκε από την Ατλέτικο Μπιλμπάο με σκορ 2:3 +1 Η ήισ INTJ ADP Case=Acc|Gender=Neut|Number=Sing 2 det _ _ +2 Μάντσεστερ μάντσεσο PUNCT DET Case=Gen|Gender=Masc|Number=Sing|Person=3|PronType=Rel 4 nsubj _ _ +3 Γιουνάιτεντ γιουνάιτο INTJ ADP Foreign=Yes 2 flat _ _ +4 ηττήθηκε ηττεθηκαίνω [UNK] _ Case=Gen|Definite=Def|Gender=Neut|Number=Plur|PronType=Art 0 root _ _ +5 από από]DAP[ PROPN _ [PAD] 7 case _ _ +6 την ω [UNK] ADP [PAD] 7 det _ _ +7 Ατλέτικο ατλύτι INTJ _ Case=Gen|Gender=Masc|Number=Sing|Person=3|PronType=Rel 4 obl:agent _ _ +8 Μπιλμπάο μπιλμο [UNK] _ Case=Gen|Gender=Masc|Number=Sing|Person=3|PronType=Rel 7 flat _ _ +9 με με]DAP[ PROPN ADP Case=Gen|Gender=Neut|Number=Plur 10 case _ _ +10 σκορ όεσκο INTJ DET Foreign=Yes 4 obl _ _ +11 2:3 όε2ο PROPN _ Case=Gen|Definite=Def|Gender=Neut|Number=Plur|PronType=Art 10 nmod _ _ + +# sent_id = gdt-20120309-elwikinews-5160-3 +# text = Χθες, η Μάντσεστερ Γιουνάιτεντ ηττήθηκε με σκορ 2:3 από την Ατλέτικο Μπιλμπάο, στα πλαίσια της φάσης των 16 του Γιουρόπα Λιγκ 2011-2012. +1 Χθες ήιθεσ INTJ ADP Foreign=Yes 6 advmod _ SpaceAfter=No +2 , ,]DAP[ PART ADP Case=Gen|Gender=Masc|Number=Sing|Person=3|PronType=Rel 1 punct _ _ +3 η ω PART ADP [PAD] 4 det _ _ +4 Μάντσεστερ μάντσεσο _ _ Case=Gen|Gender=Masc|Number=Sing|Person=3|PronType=Rel 6 nsubj _ _ +5 Γιουνάιτεντ γιουνάιτο INTJ _ Foreign=Yes 4 flat _ _ +6 ηττήθηκε ηττήθηκε]DAP[ INTJ NUM Case=Gen|Definite=Def|Gender=Neut|Number=Plur|PronType=Art 0 root _ _ +7 με ώ PROPN NUM [PAD] 8 case _ _ +8 σκορ όεσκο INTJ NUM Case=Acc|Gender=Neut|Number=Sing 6 obl _ _ +9 2:3 2 PROPN _ Case=Acc|Gender=Neut|Number=Sing 8 nmod _ _ +10 από από]DAP[ PROPN _ [PAD] 12 case _ _ +11 την ω [UNK] _ [PAD] 12 det _ _ +12 Ατλέτικο ατλέτιόσ INTJ _ Case=Gen|Gender=Masc|Number=Sing|Person=3|PronType=Rel 6 obl:agent _ _ +13 Μπιλμπάο μπιλμο PROPN _ Case=Gen|Gender=Masc|Number=Sing|Person=3|PronType=Rel 12 flat _ SpaceAfter=No +14 , ,]DAP[ PART _ Case=Gen|Gender=Masc|Number=Sing|Person=3|PronType=Rel 17 punct _ _ +15-16 στα ω DET _ [PAD] _ _ _ _ +15 σ ο [UNK] _ Case=Gen|Gender=Masc|Number=Sing|Person=3|PronType=Rel 17 case _ _ +16 τα τα]DAP[ SYM _ Case=Gen|Gender=Masc|Number=Sing|Person=3|PronType=Rel 17 det _ _ +17 πλαίσια όλείσο INTJ _ [PAD] 6 obl _ _ +18 της τή INTJ DET Case=Gen|Gender=Masc|Number=Sing|Person=3|PronType=Rel 19 det _ _ +19 φάσης ω ADV ADJ Case=Gen|Gender=Neut|Number=Plur 17 nmod _ _ +20 των ω AUX DET Aspect=Perf|Mood=Ind|Number=Sing|Person=3|Tense=Past|VerbForm=Fin|Voice=Pass 21 det _ _ +21 16 16]DAP[ ADV ADP Aspect=Perf|Mood=Ind|Number=Sing|Person=3|Tense=Past|VerbForm=Fin|Voice=Pass 19 nmod _ _ +22 του του]DAP[ [UNK] DET Case=Gen|Gender=Masc|Number=Sing|Person=3|PronType=Rel 23 det _ _ +23 Γιουρόπα γιουρο INTJ ADP Foreign=Yes 21 nmod _ _ +24 Λιγκ ι INTJ DET Foreign=Yes 23 flat _ _ +25 2011-2012 20ό1ε20ο PROPN [PAD] Case=Gen|Gender=Masc|Number=Sing|Person=3|PronType=Rel 23 nmod _ SpaceAfter=No +26 . .]DAP[ INTJ ADP Case=Gen|Gender=Neut|Number=Plur|NumType=Card 6 punct _ _ + +# sent_id = gdt-20120309-elwikinews-5160-4 +# text = Το σκορ του αγώνα άνοιξε ο Γουέν Ρούνι στο 22ο λεπτό, ωστόσο οι φιλοξενούμενοι ισοφάρισαν με τον Λλορέντε στο 44'. +1 Το ήιτσ ADP ADP Case=Acc|Gender=Neut|Number=Sing 2 det _ _ +2 σκορ όεσκο INTJ _ Foreign=Yes 5 obj _ _ +3 του όετο ADP _ Case=Gen|Gender=Masc|Number=Sing|Person=3|PronType=Rel 4 det _ _ +4 αγώνα ω PROPN ADP Case=Acc|Definite=Def|Gender=Fem|Number=Plur|PronType=Art 2 nmod _ _ +5 άνοιξε άνοιξε]DAP[ INTJ _ [PAD] 0 root _ _ +6 ο ο]DAP[ PROPN _ [PAD] 7 det _ _ +7 Γουέν όεουο PROPN ADP Foreign=Yes 5 nsubj _ _ +8 Ρούνι όεούο PROPN _ Case=Gen|Gender=Masc|Number=Sing|Person=3|PronType=Rel 7 flat _ _ +9-10 στο στο]DAP[ PROPN _ [PAD] _ _ _ _ +9 σ ω PRON _ Foreign=Yes 12 case _ _ +10 το το]DAP[ PART _ Case=Gen|Gender=Masc|Number=Sing|Person=3|PronType=Rel 12 det _ _ +11 22ο 22ο]DAP[ PROPN _ Case=Gen|Gender=Neut|Number=Plur|NumType=Card 12 amod _ _ +12 λεπτό λεπή PROPN _ Case=Gen|Gender=Masc|Number=Sing|Person=3|PronType=Rel 5 obl _ SpaceAfter=No +13 , ,]DAP[ PART ADP Case=Gen|Gender=Masc|Number=Sing|Person=3|PronType=Rel 17 punct _ _ +14 ωστόσο ωετόσαίνω PROPN ADP Case=Gen|Gender=Masc|Number=Sing|Person=3|PronType=Rel 17 cc _ _ +15 οι ο PART _ Foreign=Yes 16 det _ _ +16 φιλοξενούμενοι φιλοξενόύεενο PART ADP Foreign=Yes 17 nsubj _ _ +17 ισοφάρισαν ισοόάεισο PROPN _ Aspect=Imp|Mood=Ind|Number=Sing|Person=3|Tense=Pres|VerbForm=Fin|Voice=Act 5 conj _ _ +18 με ω PROPN _ [PAD] 20 case _ _ +19 τον ο PROPN ADP NumType=Card 20 det _ _ +20 Λλορέντε λλοεένταίνω PROPN ADP Case=Gen|Gender=Masc|Number=Sing|Person=3|PronType=Rel 17 obl _ _ +21-22 στο όεσο PROPN _ Case=Gen|Gender=Masc|Number=Sing|Person=3|PronType=Rel _ _ _ _ +21 σ ω PRON _ Foreign=Yes 23 case _ _ +22 το όεο SYM _ Case=Gen|Gender=Masc|Number=Sing|Person=3|PronType=Rel 23 det _ _ +23 44' όε4ο SYM _ Case=Nom|Gender=Neut|Number=Sing|Person=3|PronType=Rel 17 obl _ SpaceAfter=No +24 . όεο PROPN ADP Case=Gen|Gender=Neut|Number=Plur|NumType=Card 5 punct _ _ + +# sent_id = gdt-20120309-elwikinews-5160-5 +# text = Στο δεύτερο ημίχρονο, η Ατλέτικο πέτυχε δύο τέρματα με τους Όσκαρ ντε Μάρκος (71ο λεπτό) και Ικέρ Μουνιάιν (90ο λεπτό). +1-2 Στο ήιστσ PART ADP [PAD] _ _ _ _ +1 Σ ω PUNCT DET Case=Gen|Gender=Masc|Number=Sing|Person=3|PronType=Rel 4 case _ _ +2 το ή PUNCT ADP Case=Gen|Gender=Masc|Number=Sing|Person=3|PronType=Rel 4 det _ _ +3 δεύτερο δεετεραίνω PUNCT ADP Foreign=Yes 4 amod _ _ +4 ημίχρονο ημίχρο ADV ADP Foreign=Yes 8 obl _ SpaceAfter=No +5 , ή PUNCT ADP Foreign=Yes 4 punct _ _ +6 η ω DET ADP [PAD] 7 det _ _ +7 Ατλέτικο ατλέτο _ ADP Foreign=Yes 8 nsubj _ _ +8 πέτυχε όπετυο ADJ ADP NumType=Card 0 root _ _ +9 δύο ώ PROPN ADP NumType=Card 10 nummod _ _ +10 τέρματα όέεμαο PROPN ADP NumType=Card 8 obj _ _ +11 με ω PROPN ADP NumType=Card 13 case _ _ +12 τους τοή DET ADP Foreign=Yes 13 det _ _ +13 Όσκαρ ύσι SYM ADP Foreign=Yes 8 obl _ _ +14 ντε ω PROPN ADP NumType=Card 13 flat _ _ +15 Μάρκος μάρκ PROPN _ Case=Gen|Gender=Masc|Number=Sing 13 flat _ _ +16 ( εαίνω SYM _ Case=Gen|Definite=Def|Gender=Fem|Number=Sing|PronType=Art 18 punct _ SpaceAfter=No +17 71ο όε7ο PROPN ADP NumType=Card 18 amod _ _ +18 λεπτό λεπή PROPN ADP Case=Gen|Gender=Masc|Number=Sing|Person=3|PronType=Rel 8 obl _ SpaceAfter=No +19 ) ομ) PROPN ADP Case=Gen|Gender=Masc|Number=Sing|Person=3|PronType=Rel 18 punct _ _ +20 και όεκο SYM ADP Case=Gen|Definite=Def|Gender=Neut|Number=Plur|PronType=Art 21 cc _ _ +21 Ικέρ όεικο SYM ADP Foreign=Yes 8 conj _ _ +22 Μουνιάιν μόυειάο SYM _ NumType=Card 21 flat _ _ +23 ( όεο SYM _ Case=Gen|Definite=Def|Gender=Fem|Number=Sing|PronType=Art 25 punct _ SpaceAfter=No +24 90ο όε9ο _ ADP NumType=Card 25 amod _ _ +25 λεπτό λεπή PROPN ADP Case=Gen|Gender=Masc|Number=Sing|Person=3|PronType=Rel 21 orphan _ SpaceAfter=No +26 ) όεο PROPN ADP Case=Gen|Definite=Def|Gender=Neut|Number=Plur|PronType=Art 25 punct _ SpaceAfter=No +27 . όεο INTJ ADP Case=Acc|Gender=Neut|Number=Plur|NumType=Card 8 punct _ _ + +# sent_id = gdt-20120309-elwikinews-5160-6 +# text = Ωστόσο, ο Γουέν Ρούνι με πέναλτι μείωσε το σκορ για την Μάντσεστερ. +1 Ωστόσο ωετόσαίνω DET ADP Foreign=Yes 8 cc _ SpaceAfter=No +2 , ω PART ADP Case=Gen|Gender=Masc|Number=Sing|Person=3|PronType=Rel 1 punct _ _ +3 ο ω PART ADP Foreign=Yes 4 det _ _ +4 Γουέν όεουο PROPN ADP Foreign=Yes 8 nsubj _ _ +5 Ρούνι όεούο PROPN [PAD] Foreign=Yes 4 flat _ _ +6 με όεο INTJ ADP Case=Acc|Gender=Neut|Number=Sing 7 case _ _ +7 πέναλτι όέεαλο INTJ ADP Case=Gen|Gender=Masc|Number=Sing|Person=3|PronType=Rel 8 obl _ _ +8 μείωσε μείωσαίνω PROPN ADP Case=Gen|Gender=Masc|Number=Sing|Person=3|PronType=Rel 0 root _ _ +9 το εταίνω PROPN ADP Case=Gen|Gender=Masc|Number=Sing|Person=3|PronType=Rel 10 det _ _ +10 σκορ εσκοαίνω PROPN ADP Foreign=Yes 8 obj _ _ +11 για ω PROPN ADP [PAD] 13 case _ _ +12 την ετηαίνω PROPN ADP Foreign=Yes 13 det _ _ +13 Μάντσεστερ μάντσεσο _ ADP Foreign=Yes 8 obl _ SpaceAfter=No +14 . ω PART ADP Foreign=Yes 8 punct _ _ + +# sent_id = gdt-20120309-elwikinews-5160-7 +# text = Οι δύο αντίπαλοι θα ξανασυναντηθούν στις 15 Μαρτίου στο Στάδιο «Σαν Μαμές», με τους Ισπανούς να χρειάζονται νίκη και ισοπαλία και με τους κόκκινους διαβόλους να χρειάζονται νίκη με διαφορά δύο τερμάτων. +1 Οι ήιοσ PART ADP [PAD] 3 det _ _ +2 δύο όεδο PUNCT [PAD] Foreign=Yes 3 nummod _ _ +3 αντίπαλοι ανόίεαλο PUNCT ADP Foreign=Yes 5 nsubj:pass _ _ +4 θα ήιθσ PUNCT _ Case=Gen|Gender=Masc|Number=Sing|Person=3|PronType=Rel 5 aux _ _ +5 ξανασυναντηθούν ξανασυναότεθοο AUX _ Foreign=Yes 0 root _ _ +6-7 στις όεστο SYM ADP [PAD] _ _ _ _ +6 σ ω ADV ADP Case=Gen|Definite=Def|Gender=Fem|Number=Sing|PronType=Art 8 case _ _ +7 τις ω SYM ADP Case=Gen|Definite=Def|Gender=Fem|Number=Sing|PronType=Art 8 det _ _ +8 15 15]DAP[ ADV ADP Case=Nom|Definite=Def|Gender=Masc|Number=Sing|PronType=Art 5 obl _ _ +9 Μαρτίου μαρτο SYM ADP Case=Gen|Definite=Def|Gender=Neut|Number=Plur|PronType=Art 8 nmod _ _ +10-11 στο στο]DAP[ SYM ADP Case=Gen|Gender=Masc|Number=Sing|Person=3|PronType=Rel _ _ _ _ +10 σ ω [UNK] PROPN Case=Gen|Gender=Masc|Number=Sing|Person=3|PronType=Rel 12 case _ _ +11 το το]DAP[ SYM ADP Case=Gen|Gender=Masc|Number=Sing|Person=3|PronType=Rel 12 det _ _ +12 Στάδιο στάδ PROPN ADP Aspect=Perf|Mood=Ind|Number=Sing|Person=3|Tense=Past|VerbForm=Fin|Voice=Pass 5 obl _ _ +13 « SYM ADP [PAD] 14 punct _ SpaceAfter=No +14 Σαν σ SYM ADP Foreign=Yes 12 flat _ _ +15 Μαμές μαμ SCONJ ADP Foreign=Yes 12 flat _ SpaceAfter=No +16 » »]DAP[ PROPN ADP Foreign=Yes 15 punct _ SpaceAfter=No +17 , ,]DAP[ PART ADP _ 20 punct _ _ +18 με εμαίνω PROPN ADP [PAD] 20 case _ _ +19 τους τοή PROPN ADP Foreign=Yes 20 det _ _ +20 Ισπανούς ιήπαιούσ SYM ADP Foreign=Yes 5 obl _ _ +21 να εναίνω SYM _ Case=Gen|Gender=Masc|Number=Sing|Person=3|PronType=Rel 22 aux _ _ +22 χρειάζονται χρειόζεντο SYM _ NumType=Card 20 acl _ _ +23 νίκη όενίο PROPN ADP Case=Gen|Gender=Masc|Number=Sing|Person=3|PronType=Rel 22 obj _ _ +24 και όεκο PROPN _ NumType=Card 25 cc _ _ +25 ισοπαλία ιόοεαλο DET ADP Foreign=Yes 23 conj _ _ +26 και κή SYM _ Case=Gen|Gender=Masc|Number=Sing|Person=3|PronType=Rel 30 cc _ _ +27 με εμαίνω [UNK] ADP [PAD] 30 case _ _ +28 τους τοή SYM ADP Foreign=Yes 30 det _ _ +29 κόκκινους κόκκινουσ]DAP[ SYM ADP Case=Gen|Definite=Def|Gender=Fem|Number=Sing|PronType=Art 30 amod _ _ +30 διαβόλους διαβόλουσ]DAP[ SYM DET Foreign=Yes 20 conj _ _ +31 να όεο SYM _ Case=Gen|Gender=Masc|Number=Sing|Person=3|PronType=Rel 32 aux _ _ +32 χρειάζονται χρειόζεντο SYM _ NumType=Card 30 acl _ _ +33 νίκη όενίο PROPN ADP NumType=Card 32 obj _ _ +34 με όεο INTJ ADP Case=Gen|Gender=Masc|Number=Sing|Person=3|PronType=Rel 35 case _ _ +35 διαφορά όιεφοο SYM ADP [PAD] 33 nmod _ _ +36 δύο όεδο SYM [PAD] Case=Gen|Definite=Def|Gender=Neut|Number=Plur|PronType=Art 37 nummod _ _ +37 τερμάτων τερμάτων]DAP[ PROPN ADP Case=Gen|Definite=Def|Gender=Fem|Number=Sing|PronType=Art 35 nmod _ SpaceAfter=No +38 . .]DAP[ INTJ ADP Case=Acc|Gender=Neut|Number=Plur|NumType=Card 5 punct _ _ + +# newdoc id = gdt-2005XXXX-ert-tourism_menoume_ellada_axerontas +# sent_id = gdt-2005XXXX-ert-tourism_menoume_ellada_axerontas-1 +# text = Στον ποταμό Αχέροντα. +1-2 Στον στον_ PART ADP [PAD] _ _ _ _ +1 Σ ο SYM PROPN Case=Gen|Gender=Masc|Number=Sing|Person=3|PronType=Rel 3 case _ _ +2 τον τή SYM ADP Case=Gen|Gender=Masc|Number=Sing|Person=3|PronType=Rel 3 det _ _ +3 ποταμό ποτα INTJ ADP _ 0 root _ _ +4 Αχέροντα αχέροο SYM ADP NumType=Card 3 flat _ SpaceAfter=No +5 . ο INTJ ADP Case=Acc|Gender=Neut|Number=Plur|NumType=Card 3 punct _ _ + +# sent_id = gdt-2005XXXX-ert-tourism_menoume_ellada_axerontas-2 +# text = Ο ποταμός Αχέροντας αποτελούσε κατά τη μυθολογία το δρόμο μέσω του οποίου ο Άδης μετέφερε τις ψυχές στο βασίλειό του, στη λίμνη Αχερουσία. +1 Ο ο_ PART ADP [PAD] 2 det _ _ +2 ποταμός ποταο INTJ ADP _ 4 nsubj _ _ +3 Αχέροντας αχέροντή PUNCT ADP Case=Gen|Gender=Neut|Number=Plur|NumType=Card 2 flat _ _ +4 αποτελούσε αποόεεούο SCONJ _ _ 0 root _ _ +5 κατά κατά]DAP[ ADJ DET [PAD] 7 case _ _ +6 τη τη]DAP[ INTJ DET Case=Gen|Gender=Masc|Number=Sing|Person=3|PronType=Rel 7 det _ _ +7 μυθολογία μυθοεογίαίνω PART DET Case=Gen|Gender=Masc|Number=Sing|Person=3|PronType=Rel 4 obl _ _ +8 το το]DAP[ PART _ Case=Acc|Gender=Neut|Number=Plur|NumType=Card 9 det _ _ +9 δρόμο δρό SYM ADP _ 4 obj _ _ +10 μέσω όεμέο SYM _ [PAD] 12 case _ _ +11 του του]DAP[ SYM ADP [PAD] 12 det _ _ +12 οποίου όοεοίο PART ADP Case=Gen|Gender=Neut|Number=Plur|NumType=Card 15 obl _ _ +13 ο ύι PART ADP [PAD] 14 det _ _ +14 Άδης άο PUNCT ADP Case=Gen|Gender=Neut|Number=Plur|NumType=Card 15 nsubj _ _ +15 μετέφερε μετέφεή SYM _ _ 9 acl:relcl _ _ +16 τις τή SYM ADP [PAD] 17 det _ _ +17 ψυχές ψυο PART _ Case=Gen|Gender=Masc|Number=Sing|Person=3|PronType=Rel 15 obj _ _ +18-19 στο στο]DAP[ SYM _ Case=Gen|Gender=Masc|Number=Sing|Person=3|PronType=Rel _ _ _ _ +18 σ ω [UNK] _ Case=Gen|Gender=Masc|Number=Sing|Person=3|PronType=Rel 20 case _ _ +19 το το]DAP[ PART PROPN Case=Gen|Gender=Masc|Number=Sing|Person=3|PronType=Rel 20 det _ _ +20 βασίλειό βασίλο INTJ ADP [PAD] 15 obl _ _ +21 του τή SYM DET [PAD] 20 nmod _ SpaceAfter=No +22 , ή PART ADP [PAD] 25 punct _ _ +23-24 στη ω SYM ADV [PAD] _ _ _ _ +23 σ ω ADJ _ Foreign=Yes 25 case _ _ +24 τη τη]DAP[ PART PROPN Case=Gen|Gender=Masc|Number=Sing|Person=3|PronType=Rel 25 det _ _ +25 λίμνη λίμνη]DAP[ [UNK] ADP [PAD] 20 nmod _ _ +26 Αχερουσία αχερω ADJ ADP Foreign=Yes 25 flat _ SpaceAfter=No +27 . .]DAP[ PART ADP Case=Acc|Gender=Neut|Number=Plur|NumType=Card 4 punct _ _ + +# sent_id = gdt-2005XXXX-ert-tourism_menoume_ellada_axerontas-3 +# text = Πηγάζει από τα ορεινά του Νομού Ιωαννίνων και έπειτα από διαδρομή 64 χιλιομέτρων εκβάλλει στο Ιόνιο Πέλαγος. +1 Πηγάζει πηω INTJ ADP _ 0 root _ _ +2 από ω INTJ PROPN [PAD] 4 case _ _ +3 τα τα]DAP[ SYM ADP [PAD] 4 det _ _ +4 ορεινά ορει SYM ADP [PAD] 1 obl _ _ +5 του τή SYM ADP [PAD] 6 det _ _ +6 Νομού νοο PROPN ADP NumType=Card 4 nmod _ _ +7 Ιωαννίνων ιωαννίνή SYM ADP Foreign=Yes 6 nmod _ _ +8 και κ INTJ PROPN [PAD] 14 cc _ _ +9 έπειτα έπώ INTJ ADP [PAD] 14 advmod _ _ +10 από αή INTJ ADP [PAD] 11 case _ _ +11 διαδρομή διαερομαίνω INTJ ADP [PAD] 9 obl _ _ +12 64 SYM ADP Case=Gen|Definite=Def|Gender=Fem|Number=Sing|PronType=Art 13 nummod _ _ +13 χιλιομέτρων χιλιόμετρο PROPN ADP [PAD] 11 nmod _ _ +14 εκβάλλει εόβελλο [UNK] ADP Case=Gen|Gender=Masc|Number=Sing|Person=3|PronType=Rel 1 conj _ _ +15-16 στο στο]DAP[ INTJ ADP Case=Gen|Gender=Masc|Number=Sing|Person=3|PronType=Rel _ _ _ _ +15 σ ο [UNK] ADP Case=Gen|Gender=Masc|Number=Sing|Person=3|PronType=Rel 18 case _ _ +16 το ο PART ADP Case=Gen|Gender=Masc|Number=Sing|Person=3|PronType=Rel 18 det _ _ +17 Ιόνιο ήιινισ SYM ADP NumType=Card 18 amod _ _ +18 Πέλαγος πέλαγ SYM ADP Case=Gen|Gender=Neut|Number=Plur|NumType=Card 14 obl _ SpaceAfter=No +19 . .]DAP[ INTJ ADP Case=Acc|Gender=Neut|Number=Plur|NumType=Card 1 punct _ _ + +# sent_id = gdt-2005XXXX-ert-tourism_menoume_ellada_axerontas-4 +# text = Ο Αχέροντας διασχίζοντας την κοιλάδα που σχηματίζεται ανάμεσα στους ορεινούς όγκους της Δυτικής Ηπείρου διέρχεται από ένα στενό φαράγγι μεταξύ των βουνών Παραμυθίας και Σουλίου, το οποίο ονομάζεται "Στενά του Αχέροντα". +1 Ο ο_ PART ADP [PAD] 2 det _ _ +2 Αχέροντας αχέρονο INTJ _ Foreign=Yes 16 nsubj _ _ +3 διασχίζοντας διασχίζονο INTJ _ Foreign=Yes 16 advcl _ _ +4 την ύι PART PROPN Case=Gen|Gender=Masc|Number=Sing|Person=3|PronType=Rel 5 det _ _ +5 κοιλάδα κοιλάδα]DAP[ INTJ ADP [PAD] 3 obj _ _ +6 που πή PART _ Case=Gen|Gender=Masc|Number=Sing|Person=3|PronType=Rel 7 nsubj:pass _ _ +7 σχηματίζεται σχηματίζετή PROPN _ Case=Gen|Gender=Masc|Number=Sing|Person=3|PronType=Rel 5 acl:relcl _ _ +8 ανάμεσα ανάμεή [UNK] ADP [PAD] 7 advmod _ _ +9-10 στους στοή SYM ADP [PAD] _ _ _ _ +9 σ ο [UNK] _ Case=Gen|Definite=Def|Gender=Fem|Number=Sing|PronType=Art 12 case _ _ +10 τους τοή SYM PROPN Case=Acc|Gender=Neut|Number=Plur|NumType=Card 12 det _ _ +11 ορεινούς ορεινο PROPN NUM Foreign=Yes 12 amod _ _ +12 όγκους όγκοή ADP ADP [PAD] 8 obl _ _ +13 της τή PROPN ADP [PAD] 15 det _ _ +14 Δυτικής δυτικήσ]DAP[ PROPN ADP Foreign=Yes 15 amod _ _ +15 Ηπείρου ηπείρου]DAP[ ADJ ADP Foreign=Yes 12 nmod _ _ +16 διέρχεται διέρχετή INTJ PROPN _ 0 root _ _ +17 από αή INTJ _ [PAD] 20 case _ _ +18 ένα ένα]DAP[ ADJ ADP [PAD] 20 det _ _ +19 στενό _ [UNK] _ [PAD] 20 amod _ _ +20 φαράγγι φαράο INTJ PROPN NumType=Card 16 obl _ _ +21 μεταξύ όμεταο PROPN _ [PAD] 23 case _ _ +22 των τή SYM ADP Case=Gen|Gender=Masc|Number=Sing|Person=3|PronType=Rel 23 det _ _ +23 βουνών όβευνο SYM ADP NumType=Card 20 nmod _ _ +24 Παραμυθίας παραμυθο SYM _ Case=Gen|Gender=Masc|Number=Sing|Person=3|PronType=Rel 23 nmod _ _ +25 και όεκο SYM PROPN Case=Gen|Definite=Def|Gender=Neut|Number=Plur|PronType=Art 26 cc _ _ +26 Σουλίου σουλο PUNCT ADP Case=Gen|Gender=Masc|Number=Sing|Person=3|PronType=Rel 24 conj _ SpaceAfter=No +27 , ύι PART ADP [PAD] 30 punct _ _ +28 το το]DAP[ PART _ [PAD] 29 det _ _ +29 οποίο ώ PART ADP [PAD] 30 nsubj:pass _ _ +30 ονομάζεται ονομάύει INTJ _ Case=Acc|Gender=Neut|Number=Plur|NumType=Card 20 acl:relcl _ _ +31 " εαίνω INTJ _ Aspect=Imp|Mood=Ind|Number=Sing|Person=3|Tense=Pres|VerbForm=Fin|Voice=Act 32 punct _ SpaceAfter=No +32 Στενά στο SYM ADP _ 30 xcomp _ _ +33 του του]DAP[ SYM PROPN Case=Gen|Gender=Masc|Number=Sing|Person=3|PronType=Rel 34 det _ _ +34 Αχέροντα αχέρονή SYM ADP NumType=Card 32 nmod _ SpaceAfter=No +35 " ή SYM _ Case=Gen|Gender=Neut|Number=Plur|NumType=Card 32 punct _ SpaceAfter=No +36 . .]DAP[ INTJ ADP Case=Acc|Gender=Neut|Number=Plur|NumType=Card 16 punct _ _ + diff --git a/tests/testdata/datasets/el_train.conllu b/tests/testdata/datasets/el_train.conllu new file mode 100644 index 0000000..49ece6c --- /dev/null +++ b/tests/testdata/datasets/el_train.conllu @@ -0,0 +1,270 @@ +# newdoc id = gdt-20120309-elwikinews-5160 +# sent_id = gdt-20120309-elwikinews-5160-1 +# text = Η Μάντσεστερ Γιουνάιτεντ ηττήθηκε από την Ατλέτικο Μπιλμπάο με σκορ 2:3 +1 Η ο DET DET Case=Nom|Definite=Def|Gender=Fem|Number=Sing|PronType=Art 2 det _ _ +2 Μάντσεστερ Μάντσεστερ X X Foreign=Yes 4 nsubj _ _ +3 Γιουνάιτεντ Γιουνάιτεντ X X Foreign=Yes 2 flat _ _ +4 ηττήθηκε ηττώμαι VERB VERB Aspect=Perf|Mood=Ind|Number=Sing|Person=3|Tense=Past|VerbForm=Fin|Voice=Pass 0 root _ _ +5 από από ADP ADP _ 7 case _ _ +6 την ο DET DET Case=Acc|Definite=Def|Gender=Fem|Number=Sing|PronType=Art 7 det _ _ +7 Ατλέτικο Ατλέτικο X X Foreign=Yes 4 obl:agent _ _ +8 Μπιλμπάο Μπιλμπάο X X Foreign=Yes 7 flat _ _ +9 με με ADP ADP _ 10 case _ _ +10 σκορ σκορ X X Foreign=Yes 4 obl _ _ +11 2:3 2:3 NUM NUM NumType=Card 10 nmod _ _ + +# sent_id = gdt-20120309-elwikinews-5160-3 +# text = Χθες, η Μάντσεστερ Γιουνάιτεντ ηττήθηκε με σκορ 2:3 από την Ατλέτικο Μπιλμπάο, στα πλαίσια της φάσης των 16 του Γιουρόπα Λιγκ 2011-2012. +1 Χθες χθες ADV ADV _ 6 advmod _ SpaceAfter=No +2 , , PUNCT PUNCT _ 1 punct _ _ +3 η ο DET DET Case=Nom|Definite=Def|Gender=Fem|Number=Sing|PronType=Art 4 det _ _ +4 Μάντσεστερ Μάντσεστερ X X Foreign=Yes 6 nsubj _ _ +5 Γιουνάιτεντ Γιουνάιτεντ X X Foreign=Yes 4 flat _ _ +6 ηττήθηκε ηττώμαι VERB VERB Aspect=Perf|Mood=Ind|Number=Sing|Person=3|Tense=Past|VerbForm=Fin|Voice=Pass 0 root _ _ +7 με με ADP ADP _ 8 case _ _ +8 σκορ σκορ NOUN NOUN Case=Acc|Gender=Neut|Number=Plur 6 obl _ _ +9 2:3 2:3 NUM NUM NumType=Card 8 nmod _ _ +10 από από ADP ADP _ 12 case _ _ +11 την ο DET DET Case=Acc|Definite=Def|Gender=Fem|Number=Sing|PronType=Art 12 det _ _ +12 Ατλέτικο Ατλέτικο X X Foreign=Yes 6 obl:agent _ _ +13 Μπιλμπάο Μπιλμπάο X X Foreign=Yes 12 flat _ SpaceAfter=No +14 , , PUNCT PUNCT _ 17 punct _ _ +15-16 στα _ _ _ _ _ _ _ _ +15 σ σε ADP ADP _ 17 case _ _ +16 τα ο DET DET Case=Acc|Gender=Neut|Number=Plur 17 det _ _ +17 πλαίσια πλαίσιο NOUN NOUN Case=Acc|Gender=Neut|Number=Plur 6 obl _ _ +18 της ο DET DET Case=Gen|Definite=Def|Gender=Fem|Number=Sing|PronType=Art 19 det _ _ +19 φάσης φάση NOUN NOUN Case=Gen|Gender=Fem|Number=Sing 17 nmod _ _ +20 των ο DET DET Case=Gen|Definite=Def|Gender=Fem|Number=Plur|PronType=Art 21 det _ _ +21 16 16 NUM NUM NumType=Card 19 nmod _ _ +22 του ο DET DET Case=Gen|Definite=Def|Gender=Neut|Number=Sing|PronType=Art 23 det _ _ +23 Γιουρόπα Γιουρόπα X X Foreign=Yes 21 nmod _ _ +24 Λιγκ Λιγκ X X Foreign=Yes 23 flat _ _ +25 2011-2012 2011-2012 NUM NUM NumType=Card 23 nmod _ SpaceAfter=No +26 . . PUNCT PUNCT _ 6 punct _ _ + +# sent_id = gdt-20120309-elwikinews-5160-4 +# text = Το σκορ του αγώνα άνοιξε ο Γουέν Ρούνι στο 22ο λεπτό, ωστόσο οι φιλοξενούμενοι ισοφάρισαν με τον Λλορέντε στο 44'. +1 Το ο DET DET Case=Acc|Definite=Def|Gender=Neut|Number=Sing|PronType=Art 2 det _ _ +2 σκορ σκορ X X Foreign=Yes 5 obj _ _ +3 του ο DET DET Case=Gen|Definite=Def|Gender=Masc|Number=Sing|PronType=Art 4 det _ _ +4 αγώνα αγώνας NOUN NOUN Case=Gen|Gender=Masc|Number=Sing 2 nmod _ _ +5 άνοιξε ανοίγω VERB VERB Aspect=Perf|Mood=Ind|Number=Sing|Person=3|Tense=Past|VerbForm=Fin|Voice=Act 0 root _ _ +6 ο ο DET DET Case=Nom|Definite=Def|Gender=Masc|Number=Sing|PronType=Art 7 det _ _ +7 Γουέν Γουέν X X Foreign=Yes 5 nsubj _ _ +8 Ρούνι Ρούνι X X Foreign=Yes 7 flat _ _ +9-10 στο _ _ _ _ _ _ _ _ +9 σ σε ADP ADP _ 12 case _ _ +10 το ο DET DET Case=Acc|Gender=Neut|Number=Sing 12 det _ _ +11 22ο 22ος ADJ ADJ Case=Acc|Gender=Neut|Number=Sing|NumType=Ord 12 amod _ _ +12 λεπτό λεπτό NOUN NOUN Case=Acc|Gender=Neut|Number=Sing 5 obl _ SpaceAfter=No +13 , , PUNCT PUNCT _ 17 punct _ _ +14 ωστόσο ωστόσο CCONJ CCONJ _ 17 cc _ _ +15 οι ο DET DET Case=Nom|Definite=Def|Gender=Masc|Number=Plur|PronType=Art 16 det _ _ +16 φιλοξενούμενοι φιλοξενούμενος ADJ ADJ Case=Nom|Gender=Masc|Number=Plur 17 nsubj _ _ +17 ισοφάρισαν ισοφαρίζω VERB VERB Aspect=Perf|Mood=Ind|Number=Plur|Person=3|Tense=Past|VerbForm=Fin|Voice=Act 5 conj _ _ +18 με με ADP ADP _ 20 case _ _ +19 τον ο DET DET Case=Acc|Definite=Def|Gender=Masc|Number=Sing|PronType=Art 20 det _ _ +20 Λλορέντε Λλορέντε X X Foreign=Yes 17 obl _ _ +21-22 στο _ _ _ _ _ _ _ _ +21 σ σε ADP ADP _ 23 case _ _ +22 το ο DET DET Case=Acc|Gender=Neut|Number=Sing 23 det _ _ +23 44' 44' NUM NUM NumType=Card 17 obl _ SpaceAfter=No +24 . . PUNCT PUNCT _ 5 punct _ _ + +# sent_id = gdt-20120309-elwikinews-5160-5 +# text = Στο δεύτερο ημίχρονο, η Ατλέτικο πέτυχε δύο τέρματα με τους Όσκαρ ντε Μάρκος (71ο λεπτό) και Ικέρ Μουνιάιν (90ο λεπτό). +1-2 Στο _ _ _ _ _ _ _ _ +1 Σ σε ADP ADP _ 4 case _ _ +2 το ο DET DET Case=Acc|Gender=Neut|Number=Sing 4 det _ _ +3 δεύτερο δεύτερος ADJ ADJ Case=Acc|Gender=Neut|Number=Sing|NumType=Ord 4 amod _ _ +4 ημίχρονο ημίχρονο NOUN NOUN Case=Acc|Gender=Neut|Number=Sing 8 obl _ SpaceAfter=No +5 , , PUNCT PUNCT _ 4 punct _ _ +6 η ο DET DET Case=Nom|Definite=Def|Gender=Fem|Number=Sing|PronType=Art 7 det _ _ +7 Ατλέτικο Ατλέτικο X X Foreign=Yes 8 nsubj _ _ +8 πέτυχε πετυχαίνω VERB VERB Aspect=Perf|Mood=Ind|Number=Sing|Person=3|Tense=Past|VerbForm=Fin|Voice=Act 0 root _ _ +9 δύο δύο NUM NUM Case=Acc|Gender=Neut|Number=Plur|NumType=Card 10 nummod _ _ +10 τέρματα τέρμα NOUN NOUN Case=Acc|Gender=Neut|Number=Plur 8 obj _ _ +11 με με ADP ADP _ 13 case _ _ +12 τους ο DET DET Case=Acc|Definite=Def|Gender=Masc|Number=Plur|PronType=Art 13 det _ _ +13 Όσκαρ Όσκαρ X X Foreign=Yes 8 obl _ _ +14 ντε ντε X X Foreign=Yes 13 flat _ _ +15 Μάρκος Μάρκος X X Foreign=Yes 13 flat _ _ +16 ( ( PUNCT PUNCT _ 18 punct _ SpaceAfter=No +17 71ο 71ος ADJ ADJ Case=Acc|Gender=Neut|Number=Sing|NumType=Ord 18 amod _ _ +18 λεπτό λεπτό NOUN NOUN Case=Acc|Gender=Neut|Number=Sing 8 obl _ SpaceAfter=No +19 ) ) PUNCT PUNCT _ 18 punct _ _ +20 και και CCONJ CCONJ _ 21 cc _ _ +21 Ικέρ Ικέρ X X Foreign=Yes 8 conj _ _ +22 Μουνιάιν Μουνιάιν X X Foreign=Yes 21 flat _ _ +23 ( ( PUNCT PUNCT _ 25 punct _ SpaceAfter=No +24 90ο 90ος ADJ ADJ Case=Acc|Gender=Neut|Number=Sing|NumType=Ord 25 amod _ _ +25 λεπτό λεπτό NOUN NOUN Case=Acc|Gender=Neut|Number=Sing 21 orphan _ SpaceAfter=No +26 ) ) PUNCT PUNCT _ 25 punct _ SpaceAfter=No +27 . . PUNCT PUNCT _ 8 punct _ _ + +# sent_id = gdt-20120309-elwikinews-5160-6 +# text = Ωστόσο, ο Γουέν Ρούνι με πέναλτι μείωσε το σκορ για την Μάντσεστερ. +1 Ωστόσο ωστόσο CCONJ CCONJ _ 8 cc _ SpaceAfter=No +2 , , PUNCT PUNCT _ 1 punct _ _ +3 ο ο DET DET Case=Nom|Definite=Def|Gender=Masc|Number=Sing|PronType=Art 4 det _ _ +4 Γουέν Γουέν X X Foreign=Yes 8 nsubj _ _ +5 Ρούνι Ρούνι X X Foreign=Yes 4 flat _ _ +6 με με ADP ADP _ 7 case _ _ +7 πέναλτι πέναλτι X X Foreign=Yes 8 obl _ _ +8 μείωσε μειώνω VERB VERB Aspect=Perf|Mood=Ind|Number=Sing|Person=3|Tense=Past|VerbForm=Fin|Voice=Act 0 root _ _ +9 το ο DET DET Case=Acc|Definite=Def|Gender=Neut|Number=Sing|PronType=Art 10 det _ _ +10 σκορ σκορ X X Foreign=Yes 8 obj _ _ +11 για για ADP ADP _ 13 case _ _ +12 την ο DET DET Case=Acc|Definite=Def|Gender=Fem|Number=Sing|PronType=Art 13 det _ _ +13 Μάντσεστερ Μάντσεστερ X X Foreign=Yes 8 obl _ SpaceAfter=No +14 . . PUNCT PUNCT _ 8 punct _ _ + +# sent_id = gdt-20120309-elwikinews-5160-7 +# text = Οι δύο αντίπαλοι θα ξανασυναντηθούν στις 15 Μαρτίου στο Στάδιο «Σαν Μαμές», με τους Ισπανούς να χρειάζονται νίκη και ισοπαλία και με τους κόκκινους διαβόλους να χρειάζονται νίκη με διαφορά δύο τερμάτων. +1 Οι ο DET DET Case=Nom|Definite=Def|Gender=Masc|Number=Plur|PronType=Art 3 det _ _ +2 δύο δύο NUM NUM Case=Nom|Gender=Masc|Number=Plur|NumType=Card 3 nummod _ _ +3 αντίπαλοι αντίπαλος ADJ ADJ Case=Nom|Gender=Masc|Number=Plur 5 nsubj:pass _ _ +4 θα θα AUX AUX _ 5 aux _ _ +5 ξανασυναντηθούν ξανασυναντώ VERB VERB Aspect=Perf|Mood=Ind|Number=Plur|Person=3|VerbForm=Fin|Voice=Pass 0 root _ _ +6-7 στις _ _ _ _ _ _ _ _ +6 σ σε ADP ADP _ 8 case _ _ +7 τις ο DET DET Case=Acc|Gender=Fem|Number=Plur 8 det _ _ +8 15 15 NUM NUM NumType=Card 5 obl _ _ +9 Μαρτίου Μάρτιος PROPN PROPN Case=Gen|Gender=Masc|Number=Sing 8 nmod _ _ +10-11 στο _ _ _ _ _ _ _ _ +10 σ σε ADP ADP _ 12 case _ _ +11 το ο DET DET Case=Acc|Gender=Neut|Number=Sing 12 det _ _ +12 Στάδιο στάδιο NOUN NOUN Case=Acc|Gender=Neut|Number=Sing 5 obl _ _ +13 « « PUNCT PUNCT _ 14 punct _ SpaceAfter=No +14 Σαν Σαν X X Foreign=Yes 12 flat _ _ +15 Μαμές Μαμές X X Foreign=Yes 12 flat _ SpaceAfter=No +16 » » PUNCT PUNCT _ 15 punct _ SpaceAfter=No +17 , , PUNCT PUNCT _ 20 punct _ _ +18 με με ADP ADP _ 20 case _ _ +19 τους ο DET DET Case=Acc|Definite=Def|Gender=Masc|Number=Plur|PronType=Art 20 det _ _ +20 Ισπανούς Ισπανός PROPN PROPN Case=Acc|Gender=Masc|Number=Plur 5 obl _ _ +21 να να AUX AUX _ 22 aux _ _ +22 χρειάζονται χρειάζομαι VERB VERB Aspect=Imp|Mood=Ind|Number=Plur|Person=3|Tense=Pres|VerbForm=Fin|Voice=Pass 20 acl _ _ +23 νίκη νίκη NOUN NOUN Case=Acc|Gender=Fem|Number=Sing 22 obj _ _ +24 και και CCONJ CCONJ _ 25 cc _ _ +25 ισοπαλία ισοπαλία NOUN NOUN Case=Acc|Gender=Fem|Number=Sing 23 conj _ _ +26 και και CCONJ CCONJ _ 30 cc _ _ +27 με με ADP ADP _ 30 case _ _ +28 τους ο DET DET Case=Acc|Definite=Def|Gender=Masc|Number=Plur|PronType=Art 30 det _ _ +29 κόκκινους κόκκινος ADJ ADJ Case=Acc|Gender=Masc|Number=Plur 30 amod _ _ +30 διαβόλους διάβολος NOUN NOUN Case=Acc|Gender=Masc|Number=Plur 20 conj _ _ +31 να να AUX AUX _ 32 aux _ _ +32 χρειάζονται χρειάζομαι VERB VERB Aspect=Imp|Mood=Ind|Number=Plur|Person=3|Tense=Pres|VerbForm=Fin|Voice=Pass 30 acl _ _ +33 νίκη νίκη NOUN NOUN Case=Acc|Gender=Fem|Number=Sing 32 obj _ _ +34 με με ADP ADP _ 35 case _ _ +35 διαφορά διαφορά NOUN NOUN Case=Acc|Gender=Fem|Number=Sing 33 nmod _ _ +36 δύο δύο NUM NUM Case=Gen|Gender=Neut|Number=Plur|NumType=Card 37 nummod _ _ +37 τερμάτων τέρμα NOUN NOUN Case=Gen|Gender=Neut|Number=Plur 35 nmod _ SpaceAfter=No +38 . . PUNCT PUNCT _ 5 punct _ _ + +# newdoc id = gdt-2005XXXX-ert-tourism_menoume_ellada_axerontas +# sent_id = gdt-2005XXXX-ert-tourism_menoume_ellada_axerontas-1 +# text = Στον ποταμό Αχέροντα. +1-2 Στον _ _ _ _ _ _ _ _ +1 Σ σε ADP ADP _ 3 case _ _ +2 τον ο DET DET Case=Acc|Gender=Masc|Number=Sing 3 det _ _ +3 ποταμό ποταμός NOUN NOUN Case=Acc|Gender=Masc|Number=Sing 0 root _ _ +4 Αχέροντα Αχέροντας PROPN PROPN Case=Acc|Gender=Neut|Number=Sing 3 flat _ SpaceAfter=No +5 . . PUNCT PUNCT _ 3 punct _ _ + +# sent_id = gdt-2005XXXX-ert-tourism_menoume_ellada_axerontas-2 +# text = Ο ποταμός Αχέροντας αποτελούσε κατά τη μυθολογία το δρόμο μέσω του οποίου ο Άδης μετέφερε τις ψυχές στο βασίλειό του, στη λίμνη Αχερουσία. +1 Ο ο DET DET Case=Nom|Definite=Def|Gender=Masc|Number=Sing|PronType=Art 2 det _ _ +2 ποταμός ποταμός NOUN NOUN Case=Nom|Gender=Masc|Number=Sing 4 nsubj _ _ +3 Αχέροντας Αχέροντας PROPN PROPN Case=Nom|Gender=Masc|Number=Sing 2 flat _ _ +4 αποτελούσε αποτελώ VERB VERB Aspect=Imp|Mood=Ind|Number=Sing|Person=3|Tense=Past|VerbForm=Fin|Voice=Act 0 root _ _ +5 κατά κατά ADP ADP _ 7 case _ _ +6 τη ο DET DET Case=Acc|Definite=Def|Gender=Fem|Number=Sing|PronType=Art 7 det _ _ +7 μυθολογία μυθολογία NOUN NOUN Case=Acc|Gender=Fem|Number=Sing 4 obl _ _ +8 το ο DET DET Case=Acc|Definite=Def|Gender=Masc|Number=Sing|PronType=Art 9 det _ _ +9 δρόμο δρόμος NOUN NOUN Case=Acc|Gender=Masc|Number=Sing 4 obj _ _ +10 μέσω μέσω ADP ADP _ 12 case _ _ +11 του ο DET DET Case=Gen|Definite=Def|Gender=Masc|Number=Sing|PronType=Art 12 det _ _ +12 οποίου οποίος PRON PRON Case=Gen|Gender=Masc|Number=Sing|Person=3|PronType=Rel 15 obl _ _ +13 ο ο DET DET Case=Nom|Definite=Def|Gender=Masc|Number=Sing|PronType=Art 14 det _ _ +14 Άδης άδης PROPN PROPN Case=Nom|Gender=Masc|Number=Sing 15 nsubj _ _ +15 μετέφερε μεταφέρω VERB VERB Aspect=Imp|Mood=Ind|Number=Sing|Person=3|Tense=Past|VerbForm=Fin|Voice=Act 9 acl:relcl _ _ +16 τις ο DET DET Case=Acc|Definite=Def|Gender=Fem|Number=Plur|PronType=Art 17 det _ _ +17 ψυχές ψυχή NOUN NOUN Case=Acc|Gender=Fem|Number=Plur 15 obj _ _ +18-19 στο _ _ _ _ _ _ _ _ +18 σ σε ADP ADP _ 20 case _ _ +19 το ο DET DET Case=Acc|Gender=Neut|Number=Sing 20 det _ _ +20 βασίλειό βασίλειο NOUN NOUN Case=Acc|Gender=Neut|Number=Sing 15 obl _ _ +21 του μου PRON PRON Case=Gen|Gender=Masc|Number=Sing|Person=3|Poss=Yes|PronType=Prs 20 nmod _ SpaceAfter=No +22 , , PUNCT PUNCT _ 25 punct _ _ +23-24 στη _ _ _ _ _ _ _ _ +23 σ σε ADP ADP _ 25 case _ _ +24 τη ο DET DET Case=Acc|Gender=Fem|Number=Sing 25 det _ _ +25 λίμνη λίμνη NOUN NOUN Case=Acc|Gender=Fem|Number=Sing 20 nmod _ _ +26 Αχερουσία Αχερουσία PROPN PROPN Case=Acc|Gender=Fem|Number=Sing 25 flat _ SpaceAfter=No +27 . . PUNCT PUNCT _ 4 punct _ _ + +# sent_id = gdt-2005XXXX-ert-tourism_menoume_ellada_axerontas-3 +# text = Πηγάζει από τα ορεινά του Νομού Ιωαννίνων και έπειτα από διαδρομή 64 χιλιομέτρων εκβάλλει στο Ιόνιο Πέλαγος. +1 Πηγάζει πηγάζω VERB VERB Aspect=Imp|Mood=Ind|Number=Sing|Person=3|Tense=Pres|VerbForm=Fin|Voice=Act 0 root _ _ +2 από από ADP ADP _ 4 case _ _ +3 τα ο DET DET Case=Acc|Definite=Def|Gender=Neut|Number=Plur|PronType=Art 4 det _ _ +4 ορεινά ορεινός ADJ ADJ Case=Acc|Gender=Neut|Number=Plur 1 obl _ _ +5 του ο DET DET Case=Gen|Definite=Def|Gender=Masc|Number=Sing|PronType=Art 6 det _ _ +6 Νομού νομός NOUN NOUN Case=Gen|Gender=Masc|Number=Sing 4 nmod _ _ +7 Ιωαννίνων Ιωάννινα PROPN PROPN Case=Gen|Gender=Neut|Number=Plur 6 nmod _ _ +8 και και CCONJ CCONJ _ 14 cc _ _ +9 έπειτα έπειτα ADV ADV _ 14 advmod _ _ +10 από από ADP ADP _ 11 case _ _ +11 διαδρομή διαδρομή NOUN NOUN Case=Acc|Gender=Fem|Number=Sing 9 obl _ _ +12 64 64 NUM NUM NumType=Card 13 nummod _ _ +13 χιλιομέτρων χιλιόμετρο NOUN NOUN Case=Gen|Gender=Neut|Number=Plur 11 nmod _ _ +14 εκβάλλει εκβάλλω VERB VERB Aspect=Imp|Mood=Ind|Number=Sing|Person=3|Tense=Pres|VerbForm=Fin|Voice=Act 1 conj _ _ +15-16 στο _ _ _ _ _ _ _ _ +15 σ σε ADP ADP _ 18 case _ _ +16 το ο DET DET Case=Acc|Gender=Neut|Number=Sing 18 det _ _ +17 Ιόνιο Ιόνιο ADJ ADJ Case=Acc|Gender=Neut|Number=Sing 18 amod _ _ +18 Πέλαγος πέλαγος NOUN NOUN Case=Acc|Gender=Neut|Number=Sing 14 obl _ SpaceAfter=No +19 . . PUNCT PUNCT _ 1 punct _ _ + +# sent_id = gdt-2005XXXX-ert-tourism_menoume_ellada_axerontas-4 +# text = Ο Αχέροντας διασχίζοντας την κοιλάδα που σχηματίζεται ανάμεσα στους ορεινούς όγκους της Δυτικής Ηπείρου διέρχεται από ένα στενό φαράγγι μεταξύ των βουνών Παραμυθίας και Σουλίου, το οποίο ονομάζεται "Στενά του Αχέροντα". +1 Ο ο DET DET Case=Nom|Definite=Def|Gender=Masc|Number=Sing|PronType=Art 2 det _ _ +2 Αχέροντας Αχέροντας PROPN PROPN Case=Nom|Gender=Masc|Number=Sing 16 nsubj _ _ +3 διασχίζοντας διασχίζω VERB VERB Aspect=Imp|VerbForm=Conv|Voice=Act 16 advcl _ _ +4 την ο DET DET Case=Acc|Definite=Def|Gender=Fem|Number=Sing|PronType=Art 5 det _ _ +5 κοιλάδα κοιλάδα NOUN NOUN Case=Acc|Gender=Fem|Number=Sing 3 obj _ _ +6 που που PRON PRON Case=Nom|Gender=Fem|Number=Sing|Person=3|PronType=Rel 7 nsubj:pass _ _ +7 σχηματίζεται σχηματίζω VERB VERB Aspect=Imp|Mood=Ind|Number=Sing|Person=3|Tense=Pres|VerbForm=Fin|Voice=Pass 5 acl:relcl _ _ +8 ανάμεσα ανάμεσα ADV ADV _ 7 advmod _ _ +9-10 στους _ _ _ _ _ _ _ _ +9 σ σε ADP ADP _ 12 case _ _ +10 τους ο DET DET Case=Acc|Gender=Masc|Number=Plur 12 det _ _ +11 ορεινούς ορεινός ADJ ADJ Case=Acc|Gender=Masc|Number=Plur 12 amod _ _ +12 όγκους όγκος NOUN NOUN Case=Acc|Gender=Masc|Number=Plur 8 obl _ _ +13 της ο DET DET Case=Gen|Definite=Def|Gender=Fem|Number=Sing|PronType=Art 15 det _ _ +14 Δυτικής δυτικός ADJ ADJ Case=Gen|Gender=Fem|Number=Sing 15 amod _ _ +15 Ηπείρου Ήπειρος PROPN PROPN Case=Gen|Gender=Fem|Number=Sing 12 nmod _ _ +16 διέρχεται διέρχομαι VERB VERB Aspect=Imp|Mood=Ind|Number=Sing|Person=3|Tense=Pres|VerbForm=Fin|Voice=Pass 0 root _ _ +17 από από ADP ADP _ 20 case _ _ +18 ένα ένας DET DET Case=Acc|Definite=Ind|Gender=Neut|Number=Sing|PronType=Art 20 det _ _ +19 στενό στενός ADJ ADJ Case=Acc|Gender=Neut|Number=Sing 20 amod _ _ +20 φαράγγι φαράγγι NOUN NOUN Case=Acc|Gender=Neut|Number=Sing 16 obl _ _ +21 μεταξύ μεταξύ ADP ADP _ 23 case _ _ +22 των ο DET DET Case=Gen|Definite=Def|Gender=Neut|Number=Plur|PronType=Art 23 det _ _ +23 βουνών βουνό NOUN NOUN Case=Gen|Gender=Neut|Number=Plur 20 nmod _ _ +24 Παραμυθίας παραμυθία PROPN PROPN Case=Gen|Gender=Fem|Number=Sing 23 nmod _ _ +25 και και CCONJ CCONJ _ 26 cc _ _ +26 Σουλίου Σούλι PROPN PROPN Case=Gen|Gender=Neut|Number=Sing 24 conj _ SpaceAfter=No +27 , , PUNCT PUNCT _ 30 punct _ _ +28 το ο DET DET Case=Nom|Definite=Def|Gender=Neut|Number=Sing|PronType=Art 29 det _ _ +29 οποίο οποίος PRON PRON Case=Nom|Gender=Neut|Number=Sing|Person=3|PronType=Rel 30 nsubj:pass _ _ +30 ονομάζεται ονομάζω VERB VERB Aspect=Imp|Mood=Ind|Number=Sing|Person=3|Tense=Pres|VerbForm=Fin|Voice=Pass 20 acl:relcl _ _ +31 " " PUNCT PUNCT _ 32 punct _ SpaceAfter=No +32 Στενά στενά NOUN NOUN Case=Nom|Gender=Neut|Number=Plur 30 xcomp _ _ +33 του ο DET DET Case=Gen|Definite=Def|Gender=Masc|Number=Sing|PronType=Art 34 det _ _ +34 Αχέροντα Αχέροντας PROPN PROPN Case=Gen|Gender=Masc|Number=Sing 32 nmod _ SpaceAfter=No +35 " " PUNCT PUNCT _ 32 punct _ SpaceAfter=No +36 . . PUNCT PUNCT _ 16 punct _ _ + diff --git a/tests/testdata/datasets/en_expected.conllu b/tests/testdata/datasets/en_expected.conllu new file mode 100644 index 0000000..4b11596 --- /dev/null +++ b/tests/testdata/datasets/en_expected.conllu @@ -0,0 +1,246 @@ +# newdoc id = weblog-juancole.com_juancole_20051126063000_ENG_20051126_063000 +# sent_id = weblog-juancole.com_juancole_20051126063000_ENG_20051126_063000-0001 +# newpar id = weblog-juancole.com_juancole_20051126063000_ENG_20051126_063000-p0001 +# text = Al-Zaman : American forces killed Shaikh Abdullah al-Ani, the preacher at the mosque in the town of Qaim, near the Syrian border. +1 Al al_ PART -LRB- Degree=Pos|NumForm=Word|NumType=Ord 0 root 0:root SpaceAfter=No +2 - PROPN -LRB- _ 3 punct 3:punct SpaceAfter=No +3 Zaman zbn ADJ -LRB- _ 1 flat 1:flat _ +4 : we PUNCT -LRB- Degree=Pos|NumForm=Word|NumType=Ord 7 punct 7:punct _ +5 American americbe DET VB _ 6 amod 6:amod _ +6 forces forci SYM VB Degree=Pos|NumForm=Word|NumType=Ord 7 nsubj 7:nsubj _ +7 killed kil PROPN -LRB- _ 1 parataxis 1:parataxis _ +8 Shaikh sha PUNCT -LRB- VerbForm=Fin 7 obj 7:obj _ +9 Abdullah abdul SYM -LRB- _ 8 flat 8:flat _ +10 al PROPN [PAD] _ 8 flat 8:flat SpaceAfter=No +11 - PROPN [PAD] VerbForm=Fin 12 punct 12:punct SpaceAfter=No +12 Ani ani]KNU[ SYM _ Case=Acc|Number=Plur|Person=1|PronType=Prs 8 flat 8:flat SpaceAfter=No +13 , i PROPN VB VerbForm=Fin 15 punct 15:punct _ +14 the ti PART -LRB- VerbForm=Fin 15 det 15:det _ +15 preacher preachi PUNCT -LRB- _ 8 appos 8:appos _ +16 at we PUNCT VB Degree=Pos|NumForm=Word|NumType=Ord 18 case 18:case _ +17 the ti PART IN Degree=Pos|NumForm=Word|NumType=Ord 18 det 18:det _ +18 mosque mosque]DAP[ INTJ -LRB- _ 15 nmod 15:nmod:at _ +19 in i SCONJ -LRB- VerbForm=Fin 21 case 21:case _ +20 the ti PART -LRB- Degree=Pos|NumForm=Word|NumType=Ord 21 det 21:det _ +21 town town PROPN -LRB- Degree=Pos|NumForm=Word|NumType=Ord 18 nmod 18:nmod:in _ +22 of i PART -LRB- VerbForm=Fin 23 case 23:case _ +23 Qaim qai PROPN -LRB- VerbForm=Fin 21 nmod 21:nmod:of SpaceAfter=No +24 , i PROPN -LRB- VerbForm=Fin 28 punct 28:punct _ +25 near newe ADJ NNP Tense=Pres|VerbForm=Part 28 case 28:case _ +26 the ti PART -LRB- Tense=Pres|VerbForm=Part 28 det 28:det _ +27 Syrian syri PROPN PRP VerbForm=Fin 28 amod 28:amod _ +28 border bordwe PROPN -LRB- VerbForm=Fin 21 nmod 21:nmod:near SpaceAfter=No +29 . i INTJ -LRB- _ 1 punct 1:punct _ + +# sent_id = weblog-juancole.com_juancole_20051126063000_ENG_20051126_063000-0002 +# text = [This killing of a respected cleric will be causing us trouble for years to come.] +1 [ [_ PART -LRB- Degree=Pos|NumForm=Word|NumType=Ord 10 punct 10:punct SpaceAfter=No +2 This th SCONJ IN Tense=Pres|VerbForm=Part 3 det 3:det _ +3 killing killi [UNK] CC _ 10 nsubj 10:nsubj _ +4 of we SYM IN _ 7 case 7:case _ +5 a i PROPN VB Number=Plur 7 det 7:det _ +6 respected respecti PART RB Degree=Pos|NumForm=Word|NumType=Ord 7 amod 7:amod _ +7 cleric cley INTJ VB Case=Acc|Number=Plur|Person=1|PronType=Prs 3 nmod 3:nmod:of _ +8 will w INTJ VBD Tense=Pres|VerbForm=Part 10 aux 10:aux _ +9 be DET IN _ 10 aux 10:aux _ +10 causing causiwe ADJ VBD VerbForm=Fin 0 root 0:root _ +11 us i SYM VBN Case=Nom|Person=2|PronType=Prs 10 iobj 10:iobj _ +12 trouble troubwe PART VB Number=Sing|PronType=Dem 10 obj 10:obj _ +13 for fi PROPN VBD PronType=Rel 14 case 14:case _ +14 years yeabe [UNK] -LRB- _ 10 obl 10:obl:for _ +15 to be DET VBD Tense=Pres|VerbForm=Part 16 mark 16:mark _ +16 come coi PUNCT VBD Number=Plur 14 acl 14:acl:to SpaceAfter=No +17 . we PART VBD Definite=Def|PronType=Art 10 punct 10:punct SpaceAfter=No +18 ] SYM -LRB- Case=Acc|Number=Plur|Person=1|PronType=Prs 10 punct 10:punct _ + +# sent_id = weblog-juancole.com_juancole_20051126063000_ENG_20051126_063000-0003 +# text = DPA: Iraqi authorities announced that they had busted up 3 terrorist cells operating in Baghdad. +1 DPA dwe ADP -LRB- _ 0 root 0:root SpaceAfter=No +2 : we ADJ -LRB- _ 5 punct 5:punct _ +3 Iraqi irawe PROPN VB VerbForm=Fin 4 amod 4:amod _ +4 authorities authoritii SYM VB Degree=Pos|NumForm=Word|NumType=Ord 5 nsubj 5:nsubj _ +5 announced announcbe ADJ VBD Degree=Pos|NumForm=Word|NumType=Ord 1 parataxis 1:parataxis _ +6 that thwe ADJ -LRB- Tense=Pres|VerbForm=Part 9 mark 9:mark _ +7 they thi ADJ VB Case=Nom|Person=2|PronType=Prs 9 nsubj 9:nsubj _ +8 had bd SYM -LRB- Degree=Pos|NumForm=Word|NumType=Ord 9 aux 9:aux _ +9 busted bustbe SYM VB _ 5 ccomp 5:ccomp _ +10 up be SYM CC _ 9 compound:prt 9:compound:prt _ +11 3 be SYM VB _ 13 nummod 13:nummod _ +12 terrorist terroribe PROPN -LRB- Degree=Pos|NumForm=Word|NumType=Ord 13 amod 13:amod _ +13 cells cells]DAP[ PUNCT CC Degree=Pos|NumForm=Word|NumType=Ord 9 obj 9:obj _ +14 operating operatiwe ADJ CC Degree=Pos|NumForm=Word|NumType=Ord 13 acl 13:acl _ +15 in be PROPN -LRB- Degree=Pos|NumForm=Word|NumType=Ord 16 case 16:case _ +16 Baghdad baghdwe ADJ -LRB- VerbForm=Fin 14 obl 14:obl:in SpaceAfter=No +17 . i PART -LRB- Case=Acc|Number=Plur|Person=1|PronType=Prs 1 punct 1:punct _ + +# sent_id = weblog-juancole.com_juancole_20051126063000_ENG_20051126_063000-0004 +# text = Two of them were being run by 2 officials of the Ministry of the Interior! +1 Two two_ PART -LRB- Degree=Pos|NumForm=Word|NumType=Ord 6 nsubj:pass 6:nsubj:pass _ +2 of we SYM VBD _ 3 case 3:case _ +3 them thwe SYM -LRB- _ 1 nmod 1:nmod:of _ +4 were w SCONJ VBG Degree=Pos|NumForm=Word|NumType=Ord 6 aux 6:aux _ +5 being bei ADJ PRP Degree=Pos|NumForm=Word|NumType=Ord 6 aux:pass 6:aux:pass _ +6 run run PROPN VBD Degree=Pos|NumForm=Word|NumType=Ord 0 root 0:root _ +7 by be PART VBD Degree=Pos|NumForm=Word|NumType=Ord 9 case 9:case _ +8 2 be SYM VBD Number=Plur 9 nummod 9:nummod _ +9 officials officials]DAP[ SYM NNS Degree=Pos|NumForm=Word|NumType=Ord 6 obl:agent 6:obl:agent _ +10 of we PART VBD VerbForm=Fin 12 case 12:case _ +11 the ti PART VBD VerbForm=Fin 12 det 12:det _ +12 Ministry ministwe PROPN VBP VerbForm=Fin 9 nmod 9:nmod:of _ +13 of be INTJ NNS VerbForm=Fin 15 case 15:case _ +14 the ti PART -LRB- VerbForm=Fin 15 det 15:det _ +15 Interior inter [UNK] -LRB- VerbForm=Fin 12 nmod 12:nmod:of SpaceAfter=No +16 ! PUNCT VBD VerbForm=Fin 6 punct 6:punct _ + +# sent_id = weblog-juancole.com_juancole_20051126063000_ENG_20051126_063000-0005 +# text = The MoI in Iraq is equivalent to the US FBI, so this would be like having J. Edgar Hoover unwittingly employ at a high level members of the Weathermen bombers back in the 1960s. +1 The the_ PART -LRB- Degree=Pos|NumForm=Word|NumType=Ord 2 det 2:det _ +2 MoI m DET VB _ 6 nsubj 6:nsubj _ +3 in be SCONJ VBD _ 4 case 4:case _ +4 Iraq irbe SYM -LRB- VerbForm=Fin 2 nmod 2:nmod:in _ +5 is INTJ IN Number=Plur 6 cop 6:cop _ +6 equivalent equivalebe INTJ -LRB- _ 0 root 0:root _ +7 to be PUNCT -LRB- _ 10 case 10:case _ +8 the twe PART -LRB- _ 10 det 10:det _ +9 US we DET -LRB- Tense=Pres|VerbForm=Part 10 compound 10:compound _ +10 FBI DET CC _ 6 obl 6:obl:to SpaceAfter=No +11 , be PART -LRB- _ 17 punct 17:punct _ +12 so we PUNCT VBD Tense=Past|VerbForm=Part 17 advmod 17:advmod _ +13 this bs DET -LRB- Tense=Pres|VerbForm=Part 17 nsubj:outer 17:nsubj:outer _ +14 would would]DAP[ ADJ VBD VerbForm=Fin 17 aux 17:aux _ +15 be be PROPN -LRB- _ 17 cop 17:cop _ +16 like libe ADJ NNPS Number=Sing|PronType=Dem 17 mark 17:mark _ +17 having having_ SYM -LRB- Degree=Pos|NumForm=Word|NumType=Ord 6 parataxis 6:parataxis _ +18 J. i PART -LRB- _ 17 obj 17:obj|22:nsubj:xsubj _ +19 Edgar edg ADV -LRB- Number=Plur 18 flat 18:flat _ +20 Hoover hoovbe PUNCT -LRB- _ 18 flat 18:flat _ +21 unwittingly unwittingbe ADJ NNP VerbForm=Fin 22 advmod 22:advmod _ +22 employ emplwe INTJ CC _ 17 xcomp 17:xcomp _ +23 at we PUNCT NNP _ 26 case 26:case _ +24 a i PUNCT CC Degree=Pos|NumForm=Word|NumType=Ord 26 det 26:det _ +25 high hibe _ NNP Degree=Pos|NumForm=Word|NumType=Ord 26 amod 26:amod _ +26 level levey PUNCT CC Degree=Pos|NumForm=Word|NumType=Ord 22 obl 22:obl:at _ +27 members members]DAP[ PUNCT -LRB- Case=Nom|Person=2|PronType=Prs 22 obj 22:obj _ +28 of be PART NNS _ 31 case 31:case _ +29 the ti PART -LRB- _ 31 det 31:det _ +30 Weathermen weathermbe PUNCT -LRB- _ 31 compound 31:compound _ +31 bombers bombers]DAP[ INTJ -LRB- Tense=Pres|VerbForm=Part 27 nmod 27:nmod:of _ +32 back babe PUNCT NNP _ 35 advmod 35:advmod _ +33 in be AUX -LRB- VerbForm=Fin 35 case 35:case _ +34 the ti PART -LRB- Tense=Pres|VerbForm=Part 35 det 35:det _ +35 1960s 19 PART -LRB- VerbForm=Fin 22 obl 22:obl:in SpaceAfter=No +36 . we PART -LRB- _ 6 punct 6:punct _ + +# sent_id = weblog-juancole.com_juancole_20051126063000_ENG_20051126_063000-0006 +# text = The third was being run by the head of an investment firm. +1 The the_ PART -LRB- Case=Acc|Number=Plur|Person=1|PronType=Prs 2 det 2:det _ +2 third tbd PUNCT VBD Number=Sing|PronType=Dem 5 nsubj:pass 5:nsubj:pass _ +3 was wbe SCONJ VB _ 5 aux 5:aux _ +4 being beibe ADJ CC _ 5 aux:pass 5:aux:pass _ +5 run run ADJ VBD _ 0 root 0:root _ +6 by be ADJ VBD Degree=Pos|NumForm=Word|NumType=Ord 8 case 8:case _ +7 the ti ADJ VBD _ 8 det 8:det _ +8 head hebe PART VBD Degree=Pos|NumForm=Word|NumType=Ord 5 obl:agent 5:obl:agent _ +9 of i PART VBD VerbForm=Fin 12 case 12:case _ +10 an i _ VBD Tense=Pres|VerbForm=Part 12 det 12:det _ +11 investment investment]DAP[ ADJ -LRB- Degree=Pos|NumForm=Word|NumType=Ord 12 compound 12:compound _ +12 firm fiwe PROPN CC _ 8 nmod 8:nmod:of SpaceAfter=No +13 . i PART VBD _ 5 punct 5:punct _ + +# sent_id = weblog-juancole.com_juancole_20051126063000_ENG_20051126_063000-0007 +# text = You wonder if he was manipulating the market with his bombing targets. +1 You you_ PART -LRB- Degree=Pos|NumForm=Word|NumType=Ord 2 nsubj 2:nsubj _ +2 wonder wonder [UNK] VB _ 0 root 0:root _ +3 if iy PROPN _ Case=Acc|Number=Plur|Person=1|PronType=Prs 6 mark 6:mark _ +4 he be SYM -LRB- _ 6 nsubj 6:nsubj _ +5 was SYM -LRB- Degree=Pos|NumForm=Word|NumType=Ord 6 aux 6:aux _ +6 manipulating manipulatiwe ADJ VB Tense=Pres|VerbForm=Part 2 ccomp 2:ccomp _ +7 the ti PART -LRB- Degree=Pos|NumForm=Word|NumType=Ord 8 det 8:det _ +8 market mar SYM -LRB- Degree=Pos|NumForm=Word|NumType=Ord 6 obj 6:obj _ +9 with wibe SYM VB Tense=Pres|VerbForm=Part 12 case 12:case _ +10 his hbe [UNK] VB _ 12 nmod:poss 12:nmod:poss _ +11 bombing bomb ADJ -LRB- Degree=Pos|NumForm=Word|NumType=Ord 12 compound 12:compound _ +12 targets targewe SYM -LRB- _ 6 obl 6:obl:with SpaceAfter=No +13 . PART VBD _ 2 punct 2:punct _ + +# sent_id = weblog-juancole.com_juancole_20051126063000_ENG_20051126_063000-0008 +# text = The cells were operating in the Ghazaliyah and al-Jihad districts of the capital. +1 The the_ PART -LRB- Case=Acc|Number=Plur|Person=1|PronType=Prs 2 det 2:det _ +2 cells celwe PUNCT CC Degree=Pos|NumForm=Word|NumType=Ord 4 nsubj 4:nsubj _ +3 were webe SCONJ VB Degree=Pos|NumForm=Word|NumType=Ord 4 aux 4:aux _ +4 operating operatiwe PUNCT CC Degree=Pos|NumForm=Word|NumType=Ord 0 root 0:root _ +5 in be SCONJ -LRB- Degree=Pos|NumForm=Word|NumType=Ord 12 case 12:case _ +6 the ti PART -LRB- Degree=Pos|NumForm=Word|NumType=Ord 12 det 12:det _ +7 Ghazaliyah ghazaliyi SYM IN VerbForm=Fin 12 compound 12:compound _ +8 and ai SYM VBD _ 11 cc 11:cc _ +9 al PROPN PRP Case=Nom|Gender=Neut|Number=Sing|Person=3|PronType=Prs 11 compound 11:compound SpaceAfter=No +10 - PROPN PRP Case=Nom|Person=2|PronType=Prs 9 punct 9:punct SpaceAfter=No +11 Jihad jihi INTJ IN _ 7 conj 7:conj:and|12:compound _ +12 districts districwe PUNCT VB Degree=Pos|NumForm=Word|NumType=Ord 4 obl 4:obl:in _ +13 of we SCONJ VB Tense=Pres|VerbForm=Part 15 case 15:case _ +14 the ti PART -LRB- _ 15 det 15:det _ +15 capital capitwe ADJ -LRB- _ 12 nmod 12:nmod:of SpaceAfter=No +16 . be INTJ VBD _ 4 punct 4:punct _ + +# sent_id = weblog-juancole.com_juancole_20051126063000_ENG_20051126_063000-0009 +# text = Although the announcement was probably made to show progress in identifying and breaking up terror cells, I don't find the news that the Baathists continue to penetrate the Iraqi government very hopeful. +1 Although although_ PART -LRB- Degree=Pos|NumForm=Word|NumType=Ord 6 mark 6:mark _ +2 the ti PUNCT VBD _ 3 det 3:det _ +3 announcement announcemebe PUNCT -LRB- Degree=Pos|NumForm=Word|NumType=Ord 6 nsubj:pass 6:nsubj:pass|8:nsubj:xsubj _ +4 was wbe SCONJ IN Number=Plur 6 aux:pass 6:aux:pass _ +5 probably proba PUNCT _ VerbForm=Fin 6 advmod 6:advmod _ +6 made mabe SYM -LRB- Degree=Pos|NumForm=Word|NumType=Ord 21 advcl 21:advcl:although _ +7 to be INTJ VBD Case=Acc|Number=Plur|Person=1|PronType=Prs 8 mark 8:mark _ +8 show shi ADJ -LRB- Tense=Past|VerbForm=Part 6 xcomp 6:xcomp _ +9 progress progrei PUNCT -LRB- Degree=Pos|NumForm=Word|NumType=Ord 8 obj 8:obj _ +10 in we INTJ VBP Tense=Pres|VerbForm=Part 11 mark 11:mark _ +11 identifying identifyiwe ADJ CC Degree=Pos|NumForm=Word|NumType=Ord 9 acl 9:acl:in _ +12 and ai _ IN Tense=Pres|VerbForm=Part 13 cc 13:cc _ +13 breaking breaki SYM CC Degree=Pos|NumForm=Word|NumType=Ord 11 conj 9:acl:in|11:conj:and _ +14 up be SYM CC Degree=Pos|NumForm=Word|NumType=Ord 13 compound:prt 13:compound:prt _ +15 terror terror]DAP[ PART PRP Degree=Pos|NumForm=Word|NumType=Ord 16 compound 16:compound _ +16 cells celbe SYM NNP Degree=Pos|NumForm=Word|NumType=Ord 11 obj 11:obj|13:obj SpaceAfter=No +17 , PART VBD Case=Acc|Number=Plur|Person=1|PronType=Prs 6 punct 6:punct _ +18 I i]DAP[ ADJ PRP Case=Nom|Person=2|PronType=Prs 21 nsubj 21:nsubj _ +19-20 don't do ADP NNP Case=Nom|Number=Sing|Person=1|PronType=Prs _ _ _ _ +19 do i PUNCT VB NumForm=Digit|NumType=Card 21 aux 21:aux _ +20 n't ni SYM VBD Case=Nom|Number=Sing|Person=1|PronType=Prs 21 advmod 21:advmod _ +21 find fiwe ADJ VBD Degree=Pos|NumForm=Word|NumType=Ord 0 root 0:root _ +22 the ti ADJ IN Tense=Pres|VerbForm=Part 23 det 23:det _ +23 news n ADJ CC Degree=Pos|NumForm=Word|NumType=Ord 21 obj 21:obj|34:nsubj:xsubj _ +24 that t INTJ NNS Tense=Pres|VerbForm=Part 27 mark 27:mark _ +25 the ti PART -LRB- Mood=Ind|Number=Sing|Person=2|Tense=Pres|VerbForm=Fin 26 det 26:det _ +26 Baathists baathisbe INTJ -LRB- VerbForm=Fin 27 nsubj 27:nsubj|29:nsubj:xsubj _ +27 continue continuy PROPN VB Tense=Past|VerbForm=Part 23 acl 23:acl:that _ +28 to we PROPN NNS Tense=Pres|VerbForm=Part 29 mark 29:mark _ +29 penetrate penetrawe [UNK] NNP VerbForm=Fin 27 xcomp 27:xcomp _ +30 the ti PART -LRB- Case=Nom|Person=2|PronType=Prs 32 det 32:det _ +31 Iraqi irawe PROPN -LRB- VerbForm=Fin 32 amod 32:amod _ +32 government governmewe SYM RB Mood=Ind|Number=Sing|Person=3|Tense=Past|VerbForm=Fin 29 obj 29:obj _ +33 very very PART PRP VerbForm=Fin 34 advmod 34:advmod _ +34 hopeful hopefwe PUNCT VB Case=Gen|Gender=Masc|Number=Sing|Person=3|Poss=Yes|PronType=Prs 21 xcomp 21:xcomp SpaceAfter=No +35 . we PART -LRB- Degree=Pos|NumForm=Word|NumType=Ord 21 punct 21:punct _ + +# sent_id = weblog-juancole.com_juancole_20051126063000_ENG_20051126_063000-0010 +# text = It reminds me too much of the ARVN officers who were secretly working for the other side in Vietnam. +1 It it_ PART -LRB- Case=Acc|Number=Plur|Person=1|PronType=Prs 2 nsubj 2:nsubj _ +2 reminds reminwe INTJ VBD Tense=Pres|VerbForm=Part 0 root 0:root _ +3 me be SYM VBD Mood=Ind|Number=Sing|Person=3|Tense=Pres|VerbForm=Fin 2 iobj 2:iobj _ +4 too tbe PART IN _ 5 advmod 5:advmod _ +5 much mube PUNCT NNP Number=Plur 2 advmod 2:advmod _ +6 of we PART VB Case=Nom|Person=2|PronType=Prs 9 case 9:case _ +7 the ti PART -LRB- Case=Nom|Person=2|PronType=Prs 9 det 9:det _ +8 ARVN ar DET VB _ 9 compound 9:compound _ +9 officers officers]DAP[ SYM VB _ 2 obl 2:obl:of|13:nsubj _ +10 who SYM VB Tense=Pres|VerbForm=Part 13 nsubj 9:ref _ +11 were were]DAP[ SYM VB Degree=Pos|NumForm=Word|NumType=Ord 13 aux 13:aux _ +12 secretly secretwe ADJ NNP Degree=Pos|NumForm=Word|NumType=Ord 13 advmod 13:advmod _ +13 working workiwe PROPN NNS Degree=Pos|NumForm=Word|NumType=Ord 9 acl:relcl 9:acl:relcl Cxn=rc-wh-nsubj +14 for fwe PROPN NNP Degree=Pos|NumForm=Word|NumType=Ord 17 case 17:case _ +15 the ti PART VB _ 17 det 17:det _ +16 other oty PROPN CC Mood=Ind|Number=Sing|Person=2|Tense=Pres|VerbForm=Fin 17 amod 17:amod _ +17 side s PROPN -LRB- _ 13 obl 13:obl:for _ +18 in be SCONJ NNP Tense=Pres|VerbForm=Part 19 case 19:case _ +19 Vietnam vietnam DET -LRB- Degree=Pos|NumForm=Word|NumType=Ord 13 obl 13:obl:in SpaceAfter=No +20 . i PART -LRB- Case=Acc|Number=Plur|Person=1|PronType=Prs 2 punct 2:punct _ + diff --git a/tests/testdata/datasets/en_train.conllu b/tests/testdata/datasets/en_train.conllu new file mode 100644 index 0000000..7ce3453 --- /dev/null +++ b/tests/testdata/datasets/en_train.conllu @@ -0,0 +1,246 @@ +# newdoc id = weblog-juancole.com_juancole_20051126063000_ENG_20051126_063000 +# sent_id = weblog-juancole.com_juancole_20051126063000_ENG_20051126_063000-0001 +# newpar id = weblog-juancole.com_juancole_20051126063000_ENG_20051126_063000-p0001 +# text = Al-Zaman : American forces killed Shaikh Abdullah al-Ani, the preacher at the mosque in the town of Qaim, near the Syrian border. +1 Al Al PROPN NNP Number=Sing 0 root 0:root SpaceAfter=No +2 - - PUNCT HYPH _ 3 punct 3:punct SpaceAfter=No +3 Zaman Zaman PROPN NNP Number=Sing 1 flat 1:flat _ +4 : : PUNCT : _ 7 punct 7:punct _ +5 American American ADJ JJ Degree=Pos 6 amod 6:amod _ +6 forces force NOUN NNS Number=Plur 7 nsubj 7:nsubj _ +7 killed kill VERB VBD Mood=Ind|Number=Plur|Person=3|Tense=Past|VerbForm=Fin 1 parataxis 1:parataxis _ +8 Shaikh Shaikh PROPN NNP Number=Sing 7 obj 7:obj _ +9 Abdullah Abdullah PROPN NNP Number=Sing 8 flat 8:flat _ +10 al al PROPN NNP Number=Sing 8 flat 8:flat SpaceAfter=No +11 - - PUNCT HYPH _ 12 punct 12:punct SpaceAfter=No +12 Ani Ani PROPN NNP Number=Sing 8 flat 8:flat SpaceAfter=No +13 , , PUNCT , _ 15 punct 15:punct _ +14 the the DET DT Definite=Def|PronType=Art 15 det 15:det _ +15 preacher preacher NOUN NN Number=Sing 8 appos 8:appos _ +16 at at ADP IN _ 18 case 18:case _ +17 the the DET DT Definite=Def|PronType=Art 18 det 18:det _ +18 mosque mosque NOUN NN Number=Sing 15 nmod 15:nmod:at _ +19 in in ADP IN _ 21 case 21:case _ +20 the the DET DT Definite=Def|PronType=Art 21 det 21:det _ +21 town town NOUN NN Number=Sing 18 nmod 18:nmod:in _ +22 of of ADP IN _ 23 case 23:case _ +23 Qaim Qaim PROPN NNP Number=Sing 21 nmod 21:nmod:of SpaceAfter=No +24 , , PUNCT , _ 28 punct 28:punct _ +25 near near ADP IN _ 28 case 28:case _ +26 the the DET DT Definite=Def|PronType=Art 28 det 28:det _ +27 Syrian Syrian ADJ JJ Degree=Pos 28 amod 28:amod _ +28 border border NOUN NN Number=Sing 21 nmod 21:nmod:near SpaceAfter=No +29 . . PUNCT . _ 1 punct 1:punct _ + +# sent_id = weblog-juancole.com_juancole_20051126063000_ENG_20051126_063000-0002 +# text = [This killing of a respected cleric will be causing us trouble for years to come.] +1 [ [ PUNCT -LRB- _ 10 punct 10:punct SpaceAfter=No +2 This this DET DT Number=Sing|PronType=Dem 3 det 3:det _ +3 killing killing NOUN NN Number=Sing 10 nsubj 10:nsubj _ +4 of of ADP IN _ 7 case 7:case _ +5 a a DET DT Definite=Ind|PronType=Art 7 det 7:det _ +6 respected respected ADJ JJ Degree=Pos 7 amod 7:amod _ +7 cleric cleric NOUN NN Number=Sing 3 nmod 3:nmod:of _ +8 will will AUX MD VerbForm=Fin 10 aux 10:aux _ +9 be be AUX VB VerbForm=Inf 10 aux 10:aux _ +10 causing cause VERB VBG Tense=Pres|VerbForm=Part 0 root 0:root _ +11 us we PRON PRP Case=Acc|Number=Plur|Person=1|PronType=Prs 10 iobj 10:iobj _ +12 trouble trouble NOUN NN Number=Sing 10 obj 10:obj _ +13 for for ADP IN _ 14 case 14:case _ +14 years year NOUN NNS Number=Plur 10 obl 10:obl:for _ +15 to to PART TO _ 16 mark 16:mark _ +16 come come VERB VB VerbForm=Inf 14 acl 14:acl:to SpaceAfter=No +17 . . PUNCT . _ 10 punct 10:punct SpaceAfter=No +18 ] ] PUNCT -RRB- _ 10 punct 10:punct _ + +# sent_id = weblog-juancole.com_juancole_20051126063000_ENG_20051126_063000-0003 +# text = DPA: Iraqi authorities announced that they had busted up 3 terrorist cells operating in Baghdad. +1 DPA DPA PROPN NNP Number=Sing 0 root 0:root SpaceAfter=No +2 : : PUNCT : _ 5 punct 5:punct _ +3 Iraqi Iraqi ADJ JJ Degree=Pos 4 amod 4:amod _ +4 authorities authority NOUN NNS Number=Plur 5 nsubj 5:nsubj _ +5 announced announce VERB VBD Mood=Ind|Number=Plur|Person=3|Tense=Past|VerbForm=Fin 1 parataxis 1:parataxis _ +6 that that SCONJ IN _ 9 mark 9:mark _ +7 they they PRON PRP Case=Nom|Number=Plur|Person=3|PronType=Prs 9 nsubj 9:nsubj _ +8 had have AUX VBD Mood=Ind|Number=Plur|Person=3|Tense=Past|VerbForm=Fin 9 aux 9:aux _ +9 busted bust VERB VBN Tense=Past|VerbForm=Part 5 ccomp 5:ccomp _ +10 up up ADP RP _ 9 compound:prt 9:compound:prt _ +11 3 3 NUM CD NumForm=Digit|NumType=Card 13 nummod 13:nummod _ +12 terrorist terrorist ADJ JJ Degree=Pos 13 amod 13:amod _ +13 cells cell NOUN NNS Number=Plur 9 obj 9:obj _ +14 operating operate VERB VBG VerbForm=Ger 13 acl 13:acl _ +15 in in ADP IN _ 16 case 16:case _ +16 Baghdad Baghdad PROPN NNP Number=Sing 14 obl 14:obl:in SpaceAfter=No +17 . . PUNCT . _ 1 punct 1:punct _ + +# sent_id = weblog-juancole.com_juancole_20051126063000_ENG_20051126_063000-0004 +# text = Two of them were being run by 2 officials of the Ministry of the Interior! +1 Two two NUM CD NumForm=Word|NumType=Card 6 nsubj:pass 6:nsubj:pass _ +2 of of ADP IN _ 3 case 3:case _ +3 them they PRON PRP Case=Acc|Number=Plur|Person=3|PronType=Prs 1 nmod 1:nmod:of _ +4 were be AUX VBD Mood=Ind|Number=Plur|Person=3|Tense=Past|VerbForm=Fin 6 aux 6:aux _ +5 being be AUX VBG Tense=Pres|VerbForm=Part 6 aux:pass 6:aux:pass _ +6 run run VERB VBN Tense=Past|VerbForm=Part|Voice=Pass 0 root 0:root _ +7 by by ADP IN _ 9 case 9:case _ +8 2 2 NUM CD NumForm=Digit|NumType=Card 9 nummod 9:nummod _ +9 officials official NOUN NNS Number=Plur 6 obl:agent 6:obl:agent _ +10 of of ADP IN _ 12 case 12:case _ +11 the the DET DT Definite=Def|PronType=Art 12 det 12:det _ +12 Ministry Ministry PROPN NNP Number=Sing 9 nmod 9:nmod:of _ +13 of of ADP IN _ 15 case 15:case _ +14 the the DET DT Definite=Def|PronType=Art 15 det 15:det _ +15 Interior Interior PROPN NNP Number=Sing 12 nmod 12:nmod:of SpaceAfter=No +16 ! ! PUNCT . _ 6 punct 6:punct _ + +# sent_id = weblog-juancole.com_juancole_20051126063000_ENG_20051126_063000-0005 +# text = The MoI in Iraq is equivalent to the US FBI, so this would be like having J. Edgar Hoover unwittingly employ at a high level members of the Weathermen bombers back in the 1960s. +1 The the DET DT Definite=Def|PronType=Art 2 det 2:det _ +2 MoI MoI PROPN NNP Number=Sing 6 nsubj 6:nsubj _ +3 in in ADP IN _ 4 case 4:case _ +4 Iraq Iraq PROPN NNP Number=Sing 2 nmod 2:nmod:in _ +5 is be AUX VBZ Mood=Ind|Number=Sing|Person=3|Tense=Pres|VerbForm=Fin 6 cop 6:cop _ +6 equivalent equivalent ADJ JJ Degree=Pos 0 root 0:root _ +7 to to ADP IN _ 10 case 10:case _ +8 the the DET DT Definite=Def|PronType=Art 10 det 10:det _ +9 US US PROPN NNP Number=Sing 10 compound 10:compound _ +10 FBI FBI PROPN NNP Number=Sing 6 obl 6:obl:to SpaceAfter=No +11 , , PUNCT , _ 17 punct 17:punct _ +12 so so ADV RB _ 17 advmod 17:advmod _ +13 this this PRON DT Number=Sing|PronType=Dem 17 nsubj:outer 17:nsubj:outer _ +14 would would AUX MD VerbForm=Fin 17 aux 17:aux _ +15 be be AUX VB VerbForm=Inf 17 cop 17:cop _ +16 like like SCONJ IN _ 17 mark 17:mark _ +17 having have VERB VBG Tense=Pres|VerbForm=Part 6 parataxis 6:parataxis _ +18 J. J. PROPN NNP Number=Sing 17 obj 17:obj|22:nsubj:xsubj _ +19 Edgar Edgar PROPN NNP Number=Sing 18 flat 18:flat _ +20 Hoover Hoover PROPN NNP Number=Sing 18 flat 18:flat _ +21 unwittingly unwittingly ADV RB _ 22 advmod 22:advmod _ +22 employ employ VERB VB VerbForm=Inf 17 xcomp 17:xcomp _ +23 at at ADP IN _ 26 case 26:case _ +24 a a DET DT Definite=Ind|PronType=Art 26 det 26:det _ +25 high high ADJ JJ Degree=Pos 26 amod 26:amod _ +26 level level NOUN NN Number=Sing 22 obl 22:obl:at _ +27 members member NOUN NNS Number=Plur 22 obj 22:obj _ +28 of of ADP IN _ 31 case 31:case _ +29 the the DET DT Definite=Def|PronType=Art 31 det 31:det _ +30 Weathermen Weathermen PROPN NNPS Number=Plur 31 compound 31:compound _ +31 bombers bomber NOUN NNS Number=Plur 27 nmod 27:nmod:of _ +32 back back ADV RB _ 35 advmod 35:advmod _ +33 in in ADP IN _ 35 case 35:case _ +34 the the DET DT Definite=Def|PronType=Art 35 det 35:det _ +35 1960s 1960s NOUN NNS Number=Ptan|NumForm=Digit|NumType=Card 22 obl 22:obl:in SpaceAfter=No +36 . . PUNCT . _ 6 punct 6:punct _ + +# sent_id = weblog-juancole.com_juancole_20051126063000_ENG_20051126_063000-0006 +# text = The third was being run by the head of an investment firm. +1 The the DET DT Definite=Def|PronType=Art 2 det 2:det _ +2 third third ADJ JJ Degree=Pos|NumForm=Word|NumType=Ord 5 nsubj:pass 5:nsubj:pass _ +3 was be AUX VBD Mood=Ind|Number=Sing|Person=3|Tense=Past|VerbForm=Fin 5 aux 5:aux _ +4 being be AUX VBG Tense=Pres|VerbForm=Part 5 aux:pass 5:aux:pass _ +5 run run VERB VBN Tense=Past|VerbForm=Part|Voice=Pass 0 root 0:root _ +6 by by ADP IN _ 8 case 8:case _ +7 the the DET DT Definite=Def|PronType=Art 8 det 8:det _ +8 head head NOUN NN Number=Sing 5 obl:agent 5:obl:agent _ +9 of of ADP IN _ 12 case 12:case _ +10 an a DET DT Definite=Ind|PronType=Art 12 det 12:det _ +11 investment investment NOUN NN Number=Sing 12 compound 12:compound _ +12 firm firm NOUN NN Number=Sing 8 nmod 8:nmod:of SpaceAfter=No +13 . . PUNCT . _ 5 punct 5:punct _ + +# sent_id = weblog-juancole.com_juancole_20051126063000_ENG_20051126_063000-0007 +# text = You wonder if he was manipulating the market with his bombing targets. +1 You you PRON PRP Case=Nom|Person=2|PronType=Prs 2 nsubj 2:nsubj _ +2 wonder wonder VERB VBP Mood=Ind|Number=Sing|Person=2|Tense=Pres|VerbForm=Fin 0 root 0:root _ +3 if if SCONJ IN _ 6 mark 6:mark _ +4 he he PRON PRP Case=Nom|Gender=Masc|Number=Sing|Person=3|PronType=Prs 6 nsubj 6:nsubj _ +5 was be AUX VBD Mood=Ind|Number=Sing|Person=3|Tense=Past|VerbForm=Fin 6 aux 6:aux _ +6 manipulating manipulate VERB VBG Tense=Pres|VerbForm=Part 2 ccomp 2:ccomp _ +7 the the DET DT Definite=Def|PronType=Art 8 det 8:det _ +8 market market NOUN NN Number=Sing 6 obj 6:obj _ +9 with with ADP IN _ 12 case 12:case _ +10 his his PRON PRP$ Case=Gen|Gender=Masc|Number=Sing|Person=3|Poss=Yes|PronType=Prs 12 nmod:poss 12:nmod:poss _ +11 bombing bombing NOUN NN Number=Sing 12 compound 12:compound _ +12 targets target NOUN NNS Number=Plur 6 obl 6:obl:with SpaceAfter=No +13 . . PUNCT . _ 2 punct 2:punct _ + +# sent_id = weblog-juancole.com_juancole_20051126063000_ENG_20051126_063000-0008 +# text = The cells were operating in the Ghazaliyah and al-Jihad districts of the capital. +1 The the DET DT Definite=Def|PronType=Art 2 det 2:det _ +2 cells cell NOUN NNS Number=Plur 4 nsubj 4:nsubj _ +3 were be AUX VBD Mood=Ind|Number=Plur|Person=3|Tense=Past|VerbForm=Fin 4 aux 4:aux _ +4 operating operate VERB VBG Tense=Pres|VerbForm=Part 0 root 0:root _ +5 in in ADP IN _ 12 case 12:case _ +6 the the DET DT Definite=Def|PronType=Art 12 det 12:det _ +7 Ghazaliyah Ghazaliyah PROPN NNP Number=Sing 12 compound 12:compound _ +8 and and CCONJ CC _ 11 cc 11:cc _ +9 al al PROPN NNP Number=Sing 11 compound 11:compound SpaceAfter=No +10 - - PUNCT HYPH _ 9 punct 9:punct SpaceAfter=No +11 Jihad Jihad PROPN NNP Number=Sing 7 conj 7:conj:and|12:compound _ +12 districts district NOUN NNS Number=Plur 4 obl 4:obl:in _ +13 of of ADP IN _ 15 case 15:case _ +14 the the DET DT Definite=Def|PronType=Art 15 det 15:det _ +15 capital capital NOUN NN Number=Sing 12 nmod 12:nmod:of SpaceAfter=No +16 . . PUNCT . _ 4 punct 4:punct _ + +# sent_id = weblog-juancole.com_juancole_20051126063000_ENG_20051126_063000-0009 +# text = Although the announcement was probably made to show progress in identifying and breaking up terror cells, I don't find the news that the Baathists continue to penetrate the Iraqi government very hopeful. +1 Although although SCONJ IN _ 6 mark 6:mark _ +2 the the DET DT Definite=Def|PronType=Art 3 det 3:det _ +3 announcement announcement NOUN NN Number=Sing 6 nsubj:pass 6:nsubj:pass|8:nsubj:xsubj _ +4 was be AUX VBD Mood=Ind|Number=Sing|Person=3|Tense=Past|VerbForm=Fin 6 aux:pass 6:aux:pass _ +5 probably probably ADV RB _ 6 advmod 6:advmod _ +6 made make VERB VBN Tense=Past|VerbForm=Part|Voice=Pass 21 advcl 21:advcl:although _ +7 to to PART TO _ 8 mark 8:mark _ +8 show show VERB VB VerbForm=Inf 6 xcomp 6:xcomp _ +9 progress progress NOUN NN Number=Sing 8 obj 8:obj _ +10 in in SCONJ IN _ 11 mark 11:mark _ +11 identifying identify VERB VBG VerbForm=Ger 9 acl 9:acl:in _ +12 and and CCONJ CC _ 13 cc 13:cc _ +13 breaking break VERB VBG VerbForm=Ger 11 conj 9:acl:in|11:conj:and _ +14 up up ADP RP _ 13 compound:prt 13:compound:prt _ +15 terror terror NOUN NN Number=Sing 16 compound 16:compound _ +16 cells cell NOUN NNS Number=Plur 11 obj 11:obj|13:obj SpaceAfter=No +17 , , PUNCT , _ 6 punct 6:punct _ +18 I I PRON PRP Case=Nom|Number=Sing|Person=1|PronType=Prs 21 nsubj 21:nsubj _ +19-20 don't _ _ _ _ _ _ _ _ +19 do do AUX VBP Mood=Ind|Number=Sing|Person=1|Tense=Pres|VerbForm=Fin 21 aux 21:aux _ +20 n't not PART RB _ 21 advmod 21:advmod _ +21 find find VERB VB VerbForm=Inf 0 root 0:root _ +22 the the DET DT Definite=Def|PronType=Art 23 det 23:det _ +23 news news NOUN NN Number=Sing 21 obj 21:obj|34:nsubj:xsubj _ +24 that that SCONJ IN _ 27 mark 27:mark _ +25 the the DET DT Definite=Def|PronType=Art 26 det 26:det _ +26 Baathists Baathist PROPN NNPS Number=Plur 27 nsubj 27:nsubj|29:nsubj:xsubj _ +27 continue continue VERB VBP Mood=Ind|Number=Plur|Person=3|Tense=Pres|VerbForm=Fin 23 acl 23:acl:that _ +28 to to PART TO _ 29 mark 29:mark _ +29 penetrate penetrate VERB VB VerbForm=Inf 27 xcomp 27:xcomp _ +30 the the DET DT Definite=Def|PronType=Art 32 det 32:det _ +31 Iraqi Iraqi ADJ JJ Degree=Pos 32 amod 32:amod _ +32 government government NOUN NN Number=Sing 29 obj 29:obj _ +33 very very ADV RB _ 34 advmod 34:advmod _ +34 hopeful hopeful ADJ JJ Degree=Pos 21 xcomp 21:xcomp SpaceAfter=No +35 . . PUNCT . _ 21 punct 21:punct _ + +# sent_id = weblog-juancole.com_juancole_20051126063000_ENG_20051126_063000-0010 +# text = It reminds me too much of the ARVN officers who were secretly working for the other side in Vietnam. +1 It it PRON PRP Case=Nom|Gender=Neut|Number=Sing|Person=3|PronType=Prs 2 nsubj 2:nsubj _ +2 reminds remind VERB VBZ Mood=Ind|Number=Sing|Person=3|Tense=Pres|VerbForm=Fin 0 root 0:root _ +3 me I PRON PRP Case=Acc|Number=Sing|Person=1|PronType=Prs 2 iobj 2:iobj _ +4 too too ADV RB _ 5 advmod 5:advmod _ +5 much much ADV RB _ 2 advmod 2:advmod _ +6 of of ADP IN _ 9 case 9:case _ +7 the the DET DT Definite=Def|PronType=Art 9 det 9:det _ +8 ARVN ARVN PROPN NNP Number=Sing 9 compound 9:compound _ +9 officers officer NOUN NNS Number=Plur 2 obl 2:obl:of|13:nsubj _ +10 who who PRON WP PronType=Rel 13 nsubj 9:ref _ +11 were be AUX VBD Mood=Ind|Number=Plur|Person=3|Tense=Past|VerbForm=Fin 13 aux 13:aux _ +12 secretly secretly ADV RB _ 13 advmod 13:advmod _ +13 working work VERB VBG Tense=Pres|VerbForm=Part 9 acl:relcl 9:acl:relcl Cxn=rc-wh-nsubj +14 for for ADP IN _ 17 case 17:case _ +15 the the DET DT Definite=Def|PronType=Art 17 det 17:det _ +16 other other ADJ JJ Degree=Pos 17 amod 17:amod _ +17 side side NOUN NN Number=Sing 13 obl 13:obl:for _ +18 in in ADP IN _ 19 case 19:case _ +19 Vietnam Vietnam PROPN NNP Number=Sing 13 obl 13:obl:in SpaceAfter=No +20 . . PUNCT . _ 2 punct 2:punct _ + diff --git a/tests/testdata/datasets/ru_expected.conllu b/tests/testdata/datasets/ru_expected.conllu new file mode 100644 index 0000000..f23a2e3 --- /dev/null +++ b/tests/testdata/datasets/ru_expected.conllu @@ -0,0 +1,255 @@ +# sent_id = 2003Anketa.xml_2 +# text = Начальник областного управления связи Семен Еремеевич был человек простой, приходил на работу всегда вовремя, здоровался с секретаршей за руку и иногда даже писал в стенгазету заметки под псевдонимом "Муха". +1 Начальник начальтьик INTJ _ Animacy=Inan|Case=Acc|Gender=Fem|Number=Sing 8 nsubj 8:nsubj _ +2 областного областнон PROPN _ Case=Ins|Degree=Pos|Gender=Fem|Number=Sing 3 amod 3:amod _ +3 управления управленая PUNCT _ Aspect=Imp|Gender=Fem|Mood=Ind|Number=Sing|Tense=Past|VerbForm=Fin|Voice=Act 1 nmod 1:nmod:gen _ +4 связи свяоз PUNCT _ Animacy=Inan|Case=Acc|Gender=Fem|Number=Sing 3 nmod 3:nmod:gen _ +5 Семен сетьен PROPN _ Animacy=Inan|Case=Acc|Gender=Fem|Number=Sing 1 appos 1:appos _ +6 Еремеевич еремеевий PROPN _ Animacy=Inan|Case=Acc|Gender=Masc|Number=Sing 5 flat:name 5:flat:name _ +7 был лн PROPN _ Case=Nom|PronType=Ind 8 cop 8:cop _ +8 человек челокн PROPN _ Case=Acc|Number=Plur|PronType=Tot 0 root 0:root _ +9 простой простьой PART _ Case=Acc|Number=Plur|PronType=Tot 8 amod 8:amod SpaceAfter=No +10 , ,н PART _ Case=Acc|Number=Plur|PronType=Tot 11 punct 11:punct _ +11 приходил приходиь PUNCT _ Case=Nom|PronType=Ind 8 conj 0:root|8:conj _ +12 на ой PROPN _ Animacy=Inan|Case=Acc|Gender=Masc|Number=Sing 13 case 13:case _ +13 работу рабун PROPN _ Mood=Cnd 11 obl 11:obl:на:acc _ +14 всегда вседсь PROPN _ Case=Acc|Number=Plur|PronType=Tot 11 advmod 11:advmod _ +15 вовремя вовртьмя SYM _ Case=Acc|Number=Plur|PronType=Tot 11 advmod 11:advmod SpaceAfter=No +16 , ,н PART _ Case=Acc|Number=Plur|PronType=Tot 17 punct 17:punct _ +17 здоровался здороваян PUNCT _ Case=Acc|Number=Plur|PronType=Tot 8 conj 0:root|8:conj _ +18 с о SYM _ Case=Ins|Degree=Pos|Gender=Fem|Number=Sing 19 case 19:case _ +19 секретаршей секретарйн SYM _ Animacy=Inan|Case=Acc|Gender=Fem|Number=Sing 17 obl 17:obl:с:ins _ +20 за оз SYM _ Animacy=Inan|Case=Loc|Gender=Neut|Number=Sing|PronType=Dem 21 case 21:case _ +21 руку рукй SYM _ Mood=Cnd 17 obl 17:obl:за:acc _ +22 и сь PART _ Case=Acc|Number=Plur|PronType=Tot 25 cc 25:cc _ +23 иногда инодсь PUNCT _ Animacy=Inan|Case=Nom|Gender=Fem|Number=Plur 25 advmod 25:advmod _ +24 даже джсь ADJ _ Animacy=Inan|Case=Nom|Gender=Fem|Number=Plur 25 advmod 25:advmod _ +25 писал пилн SYM _ Case=Acc|Number=Plur|PronType=Tot 8 conj 0:root|8:conj _ +26 в вн PROPN _ Case=Acc|Number=Plur|PronType=Tot 27 case 27:case _ +27 стенгазету стенгазтьту SYM _ Case=Acc|Number=Plur|PronType=Tot 25 obl 25:obl:в:acc _ +28 заметки замеин PROPN _ Mood=Cnd 25 obj 25:obj _ +29 под дн PART _ Animacy=Inan|Case=Acc|Gender=Fem|Number=Sing 30 case 30:case _ +30 псевдонимом псевдонимн INTJ _ Case=Acc|Degree=Pos|Gender=Fem|Number=Sing 25 obl 25:obl:под:ins _ +31 " "н PART _ Case=Acc|Degree=Pos|Gender=Fem|Number=Sing 32 punct 32:punct SpaceAfter=No +32 Муха мтьха PART _ Case=Acc|Gender=Masc|Number=Sing|Person=3|PronType=Prs 30 nmod 30:nmod:nom SpaceAfter=No +33 " ь PUNCT _ Case=Acc|Number=Plur|PronType=Tot 32 punct 32:punct SpaceAfter=No +34 . ть. INTJ _ Case=Acc|Number=Plur|PronType=Tot 8 punct 8:punct _ + +# sent_id = 2003Anketa.xml_3 +# text = В приемной его с утра ожидали посетители, - кое-кто с важными делами, а кое-кто и с такими, которые легко можно было решить в нижестоящих инстанциях, не затрудняя Семена Еремеевича. +1 В тьв PART _ Animacy=Inan|Case=Acc|Gender=Fem|Number=Sing 2 case 2:case _ +2 приемной приемтьой PUNCT _ Case=Acc|Number=Plur|PronType=Tot 6 obl 6:obl:в:loc _ +3 его он [UNK] _ Animacy=Inan|Case=Ins|Gender=Masc|Number=Sing 6 obj 6:obj _ +4 с сь ADV _ Case=Ins|Degree=Pos|Gender=Fem|Number=Sing 5 case 5:case _ +5 утра утьра ADJ _ Case=Gen|Degree=Pos|Number=Plur 6 obl 6:obl:с:gen _ +6 ожидали ожидин ADJ _ Case=Acc|Degree=Pos|Gender=Fem|Number=Sing 0 root 0:root _ +7 посетители посетитеол SYM _ Animacy=Inan|Case=Acc|Gender=Masc|Number=Sing 6 nsubj 6:nsubj SpaceAfter=No +8 , ,н PART _ Case=Acc|Number=Plur|PronType=Tot 13 punct 13:punct _ +9 - -]DAP[ PUNCT _ Case=Acc|Number=Plur|PronType=Tot 13 punct 13:punct _ +10 кое-кто кое-он PROPN _ Case=Acc|Number=Plur|PronType=Tot 13 nsubj 13:nsubj _ +11 с ая INTJ _ Case=Ins|Degree=Pos|Gender=Fem|Number=Sing 13 case 13:case _ +12 важными важныма PROPN _ Case=Acc|Number=Plur|PronType=Tot 13 amod 13:amod _ +13 делами делама PROPN _ Case=Acc|Number=Plur|PronType=Tot 7 parataxis 7:parataxis SpaceAfter=No +14 , ,н PART _ Case=Acc|Number=Plur|PronType=Tot 19 punct 19:punct _ +15 а ой [UNK] _ Case=Acc|Degree=Pos|Gender=Fem|Number=Sing 19 cc 19:cc _ +16 кое-кто кое-он PROPN _ Case=Ins|Degree=Pos|Number=Plur 19 nsubj 19:nsubj _ +17 и сь PART _ Case=Nom|Degree=Pos|Number=Plur 19 advmod 19:advmod _ +18 с ая SYM _ Case=Nom|Degree=Pos|Number=Plur 19 case 19:case _ +19 такими такой PRON _ Case=Gen|Degree=Pos|Number=Plur 13 det 13:det|25:obj SpaceAfter=No +20 , ть, PART _ Case=Acc|Number=Plur|PronType=Tot 23 punct 23:punct _ +21 которые которые_ PART _ Animacy=Inan|Case=Loc|Gender=Neut|Number=Sing|PronType=Dem 25 obj 19:ref _ +22 легко легко_ PROPN _ Case=Nom|PronType=Ind 23 advmod 23:advmod _ +23 можно можнй PROPN _ Mood=Cnd 19 acl:relcl 19:acl:relcl _ +24 было быой PROPN _ Case=Acc|Number=Plur|PronType=Tot 23 cop 23:cop _ +25 решить решитй [UNK] _ Animacy=Inan|Case=Acc|Gender=Masc|Number=Sing 23 csubj 23:csubj _ +26 в тьв PROPN _ Case=Acc|Number=Plur|PronType=Tot 28 case 28:case _ +27 нижестоящих нижестоятьих SCONJ _ Aspect=Imp|Mood=Ind|Number=Plur|Tense=Past|VerbForm=Fin|Voice=Act 28 amod 28:amod _ +28 инстанциях инстанцияй [UNK] _ Mood=Cnd 25 obl 25:obl:в:loc SpaceAfter=No +29 , ть, PART _ Case=Acc|Number=Plur|PronType=Tot 31 punct 31:punct _ +30 не тьне [UNK] _ Animacy=Inan|Case=Nom|Gender=Fem|Number=Sing 31 advmod 31:advmod _ +31 затрудняя затрудтьяя ADJ _ Case=Acc|Degree=Pos|Gender=Fem|Number=Sing 25 advcl 25:advcl _ +32 Семена семтьна ADJ _ Animacy=Inan|Case=Acc|Gender=Fem|Number=Sing 31 obl 31:obl:gen _ +33 Еремеевича еремеевтьча ADJ _ Animacy=Inan|Case=Acc|Gender=Masc|Number=Sing 32 flat:name 32:flat:name SpaceAfter=No +34 . ть. PART _ Animacy=Inan|Case=Acc|Gender=Fem|Number=Sing 6 punct 6:punct _ + +# sent_id = 2003Anketa.xml_4 +# text = Однако стиль работы Семена Еремеевича заключался в том, чтобы принимать всех желающих и лично вникать в дело. +1 Однако однтько PART _ Animacy=Inan|Case=Acc|Gender=Fem|Number=Sing 6 advmod 6:advmod _ +2 стиль сттьль PUNCT _ Animacy=Inan|Case=Acc|Gender=Fem|Number=Sing 6 nsubj 6:nsubj _ +3 работы рабтьты PUNCT _ Animacy=Inan|Case=Acc|Gender=Fem|Number=Sing 2 nmod 2:nmod:gen _ +4 Семена семтьна SCONJ _ Animacy=Inan|Case=Acc|Gender=Fem|Number=Sing 3 nmod 3:nmod:gen _ +5 Еремеевича еремеевтьча PUNCT _ Animacy=Inan|Case=Acc|Gender=Fem|Number=Sing 4 flat:name 4:flat:name _ +6 заключался заключаян PROPN _ Case=Acc|Number=Plur|PronType=Tot 0 root 0:root _ +7 в вн PROPN _ Case=Acc|Number=Plur|PronType=Tot 8 case 8:case _ +8 том мн PROPN _ Case=Acc|Number=Plur|PronType=Tot 6 obl 6:obl:в:loc SpaceAfter=No +9 , ,н PROPN _ Case=Acc|Number=Plur|PronType=Tot 11 punct 11:punct _ +10 чтобы чтын PUNCT _ Case=Acc|Degree=Pos|Gender=Fem|Number=Sing 11 mark 11:mark _ +11 принимать принимтьть PUNCT _ Case=Nom|PronType=Ind 8 advcl 8:advcl:чтобы _ +12 всех втьех SYM _ Case=Nom|PronType=Ind 13 det 13:det _ +13 желающих желаютьих SYM _ Case=Acc|Number=Plur|PronType=Tot 11 xcomp 11:xcomp _ +14 и ь SYM _ Case=Acc|Number=Plur|PronType=Tot 16 cc 16:cc _ +15 лично личон SYM _ Case=Acc|Number=Plur|PronType=Tot 16 advmod 16:advmod _ +16 вникать вникаая [UNK] _ Case=Acc|Number=Plur|PronType=Tot 11 conj 8:advcl:чтобы|11:conj _ +17 в й PART _ Case=Acc|Number=Plur|PronType=Tot 18 case 18:case _ +18 дело деол [UNK] _ Animacy=Inan|Case=Acc|Gender=Fem|Number=Sing 16 obl 16:obl:в:acc SpaceAfter=No +19 . ть. INTJ _ Animacy=Inan|Case=Nom|Gender=Fem|Number=Plur 6 punct 6:punct _ + +# sent_id = 2003Anketa.xml_5 +# text = Приемная была обставлена просто, но по-деловому. +1 Приемная приемтьая INTJ _ Case=Acc|Number=Plur|PronType=Tot 3 nsubj:pass 3:nsubj:pass _ +2 была бан SCONJ _ Case=Nom|PronType=Ind 3 aux:pass 3:aux:pass _ +3 обставлена обставлан ADJ _ Case=Nom|PronType=Ind 0 root 0:root _ +4 просто протьто [UNK] _ Case=Acc|Number=Plur|PronType=Tot 3 advmod 3:advmod SpaceAfter=No +5 , ,н PART _ Case=Acc|Number=Plur|PronType=Tot 7 punct 7:punct _ +6 но он PROPN _ Case=Acc|PronType=Int,Rel 7 cc 7:cc _ +7 по-деловому по-деловтьму PART _ Case=Nom|PronType=Ind 4 conj 3:advmod|4:conj SpaceAfter=No +8 . ть. PART _ Animacy=Inan|Case=Nom|Gender=Fem|Number=Plur 3 punct 3:punct _ + +# sent_id = 2003Anketa.xml_6 +# text = У двери стоял стол секретарши, на столе - пишущая машинка с широкой кареткой. +1 У тьу PART _ Animacy=Inan|Case=Acc|Gender=Fem|Number=Sing 2 case 2:case _ +2 двери двин PUNCT _ Case=Acc|Number=Plur|PronType=Tot 3 obl 3:obl:у:gen _ +3 стоял стой [UNK] _ Case=Acc|Number=Plur|PronType=Tot 0 root 0:root _ +4 стол стоа INTJ _ Animacy=Inan|Case=Gen|Gender=Fem|Number=Plur 3 nsubj 3:nsubj _ +5 секретарши секретаин PUNCT _ Mood=Cnd 4 nmod 4:nmod:gen SpaceAfter=No +6 , ь PART _ Case=Acc|Number=Plur|PronType=Tot 8 punct 8:punct _ +7 на нь PUNCT _ Aspect=Imp|Gender=Masc|Mood=Ind|Number=Sing|Tense=Past|VerbForm=Fin|Voice=Act 8 case 8:case _ +8 столе стен [UNK] _ Mood=Cnd 3 conj 0:root|3:conj _ +9 - ь PART _ Case=Acc|Number=Plur|PronType=Tot 11 punct 11:punct _ +10 пишущая пишуян INTJ _ Case=Acc|Number=Plur|PronType=Tot 11 amod 11:amod _ +11 машинка машиан PROPN _ Case=Acc|Number=Plur|PronType=Tot 8 nsubj 8:nsubj _ +12 с сн SCONJ _ Case=Acc|Number=Plur|PronType=Tot 14 case 14:case _ +13 широкой широой PART _ Case=Acc|Number=Plur|PronType=Tot 14 amod 14:amod _ +14 кареткой каретйн PART _ Animacy=Inan|Case=Acc|Gender=Fem|Number=Sing 11 nmod 11:nmod:с:ins SpaceAfter=No +15 . ть. PART _ Case=Acc|Number=Plur|PronType=Tot 3 punct 3:punct _ + +# sent_id = 2003Anketa.xml_7 +# text = В углу висел репродуктор и играло радио для развлечения ожидающих и еще для того, чтобы заглушать голос начальника, доносившийся из кабинета, так как, бесспорно, среди посетителей могли находиться и случайные люди. +1 В тьв PART _ Animacy=Inan|Case=Acc|Gender=Fem|Number=Sing 2 case 2:case _ +2 углу уун SYM _ Case=Acc|Number=Plur|PronType=Tot 3 obl 3:obl:в:loc _ +3 висел висеь [UNK] _ Aspect=Imp|Mood=Ind|Number=Plur|Tense=Past|VerbForm=Fin|Voice=Act 0 root 0:root _ +4 репродуктор репродукрн [UNK] _ Mood=Cnd 3 nsubj 3:nsubj _ +5 и а PART _ Case=Acc|Number=Plur|PronType=Tot 6 cc 6:cc _ +6 играло играой PUNCT _ Animacy=Inan|Case=Ins|Gender=Masc|Number=Sing 3 conj 0:root|3:conj _ +7 радио раон INTJ _ Animacy=Inan|Case=Acc|Gender=Masc|Number=Sing 6 nsubj 6:nsubj _ +8 для дла PROPN _ Case=Acc|Number=Plur|PronType=Tot 9 case 9:case _ +9 развлечения развлечеян INTJ _ Case=Nom|PronType=Ind 6 obl 6:obl:для:gen _ +10 ожидающих ожидаютьих SYM _ Case=Acc|Number=Plur|PronType=Tot 9 acl 9:acl _ +11 и ь PART _ Case=Acc|Number=Plur|PronType=Tot 14 cc 14:cc _ +12 еще ещь PUNCT _ Case=Nom|Degree=Pos|Number=Plur 14 obl 14:obl _ +13 для лсь PROPN _ Case=Acc|Number=Plur|PronType=Tot 14 case 14:case _ +14 того того_ PROPN _ Case=Ins|Degree=Pos|Gender=Fem|Number=Sing 9 conj 6:obl:для:gen|9:conj SpaceAfter=No +15 , ,н PART _ Case=Acc|Number=Plur|PronType=Tot 17 punct 17:punct _ +16 чтобы чтын ADV _ Case=Acc|Degree=Pos|Gender=Fem|Number=Sing 17 mark 17:mark _ +17 заглушать заглушьн ADJ _ Animacy=Inan|Case=Nom|Gender=Fem|Number=Sing 14 advcl 14:advcl:чтобы _ +18 голос госн ADJ _ Animacy=Inan|Case=Acc|Gender=Fem|Number=Sing 17 obj 17:obj _ +19 начальника начальнан INTJ _ Case=Nom|Degree=Pos|Number=Plur 18 nmod 18:nmod:gen SpaceAfter=No +20 , ,н PART _ Case=Acc|Number=Plur|PronType=Tot 21 punct 21:punct _ +21 доносившийся доносившиян SYM _ Case=Acc|Number=Plur|PronType=Tot 18 acl 18:acl _ +22 из ий SYM _ Aspect=Imp|VerbForm=Inf|Voice=Act 23 case 23:case _ +23 кабинета кабинтьта SYM _ Mood=Cnd 21 obl 21:obl:из:gen SpaceAfter=No +24 , ть, PART _ Case=Acc|Number=Plur|PronType=Tot 21 punct 21:punct _ +25 так так_ PART _ Case=Acc|Number=Plur|PronType=Tot 32 mark 32:mark _ +26 как тьак INTJ _ Animacy=Anim|Case=Nom|Gender=Masc|Number=Plur 25 fixed 25:fixed SpaceAfter=No +27 , ть, PART _ Animacy=Inan|Case=Nom|Gender=Fem|Number=Plur 25 punct 25:punct _ +28 бесспорно бесспотьно INTJ _ Case=Acc|Number=Plur|PronType=Tot 32 parataxis 32:parataxis SpaceAfter=No +29 , ть, PART _ Animacy=Inan|Case=Nom|Gender=Fem|Number=Plur 28 punct 28:punct _ +30 среди сртьди [UNK] _ [PAD] 31 case 31:case _ +31 посетителей посетитетьей SYM _ Animacy=Inan|Case=Acc|Gender=Masc|Number=Sing 33 obl 33:obl:среди:gen _ +32 могли могль PUNCT _ Case=Acc|Number=Plur|PronType=Tot 17 advcl 17:advcl:так_как _ +33 находиться находитссь [UNK] _ Aspect=Imp|Gender=Masc|Mood=Ind|Number=Sing|Tense=Past|VerbForm=Fin|Voice=Mid 32 xcomp 32:xcomp _ +34 и тьи PART _ Animacy=Inan|Case=Ins|Gender=Masc|Number=Sing 35 advmod 35:advmod _ +35 случайные случайтьые PART _ Animacy=Inan|Case=Acc|Gender=Fem|Number=Sing 36 amod 36:amod _ +36 люди лин PROPN _ Animacy=Inan|Case=Ins|Gender=Masc|Number=Sing 32 nsubj 32:nsubj|33:nsubj SpaceAfter=No +37 . ть. PART _ Animacy=Inan|Case=Nom|Gender=Fem|Number=Plur 3 punct 3:punct _ + +# sent_id = 2003Anketa.xml_8 +# text = Кабинет отличался скромностью, присущей Семену Еремеевичу. +1 Кабинет кабитьет SYM _ Animacy=Inan|Case=Acc|Gender=Fem|Number=Sing 2 nsubj 2:nsubj _ +2 отличался отличалая PROPN _ Aspect=Imp|Gender=Fem|Mood=Ind|Number=Sing|Tense=Past|VerbForm=Fin|Voice=Act 0 root 0:root _ +3 скромностью скромностьью INTJ _ Case=Ins|Degree=Pos|Gender=Fem|Number=Sing 2 obl 2:obl:ins SpaceAfter=No +4 , ть, PART _ Animacy=Inan|Case=Acc|Gender=Masc|Number=Sing 5 punct 5:punct _ +5 присущей присутьей PUNCT _ Aspect=Perf|Case=Nom|Gender=Masc|Number=Sing|Tense=Past|VerbForm=Part|Voice=Pass 3 amod 3:amod _ +6 Семену семтьну ADJ _ Mood=Cnd 5 iobj 5:iobj _ +7 Еремеевичу еремеевичй PROPN _ Animacy=Inan|Case=Acc|Gender=Masc|Number=Sing 6 flat:name 6:flat:name SpaceAfter=No +8 . ть. INTJ _ Animacy=Inan|Case=Acc|Gender=Fem|Number=Sing 2 punct 2:punct _ + +# sent_id = 2003Anketa.xml_9 +# text = В глубине стоял широкий письменный стол с бронзовыми чернильницами и перед ним два кожаных кресла. +1 В тьв PART _ Animacy=Inan|Case=Acc|Gender=Fem|Number=Sing 2 case 2:case _ +2 глубине глубен PUNCT _ Case=Acc|Number=Plur|PronType=Tot 3 obl 3:obl:в:loc _ +3 стоял стой [UNK] _ Aspect=Imp|Mood=Ind|Number=Plur|Tense=Past|VerbForm=Fin|Voice=Act 0 root 0:root _ +4 широкий широйн INTJ _ Mood=Cnd 6 amod 6:amod _ +5 письменный письменйн PART _ Case=Acc|Degree=Pos|Gender=Fem|Number=Sing 6 amod 6:amod _ +6 стол стоа INTJ _ Mood=Cnd 3 nsubj 3:nsubj _ +7 с ая SCONJ _ Case=Acc|Number=Plur|PronType=Tot 9 case 9:case _ +8 бронзовыми бронзовин PART _ Case=Acc|Number=Plur|PronType=Tot 9 amod 9:amod _ +9 чернильницами чернильницин PART _ Animacy=Inan|Case=Nom|Gender=Fem|Number=Sing 6 nmod 6:nmod:с:ins _ +10 и ин INTJ _ Case=Acc|Number=Plur|PronType=Tot 12 cc 12:cc _ +11 перед перая [UNK] _ Animacy=Inan|Case=Acc|Gender=Fem|Number=Sing 12 case 12:case _ +12 ним мн SYM _ Aspect=Imp|Gender=Masc|Mood=Ind|Number=Sing|Tense=Past|VerbForm=Fin|Voice=Mid 3 conj 0:root|3:conj _ +13 два дой ADP _ Case=Acc|Number=Plur|PronType=Tot 15 nummod:gov 15:nummod:gov _ +14 кожаных кожахн SCONJ _ Case=Acc|Number=Plur|PronType=Tot 15 amod 15:amod _ +15 кресла креан [UNK] _ Animacy=Inan|Case=Acc|Gender=Fem|Number=Sing 12 nsubj 12:nsubj SpaceAfter=No +16 . й PART _ Animacy=Inan|Case=Nom|Gender=Fem|Number=Plur 3 punct 3:punct _ + +# sent_id = 2003Anketa.xml_10 +# text = Справа был стол для заседаний - длинный, накрытый зеленым сукном и с обеих сторон аккуратно заставленный стульями. +1 Справа спртьва PART _ Animacy=Inan|Case=Acc|Gender=Fem|Number=Sing 2 advmod 2:advmod _ +2 был лн DET _ Case=Acc|Number=Plur|PronType=Tot 0 root 0:root _ +3 стол стоа INTJ _ Animacy=Anim|Case=Nom|Gender=Masc|Number=Plur 2 nsubj 2:nsubj _ +4 для ян PROPN _ Animacy=Anim|Case=Nom|Gender=Masc|Number=Plur 5 case 5:case _ +5 заседаний заседатьий ADJ _ Animacy=Inan|Case=Acc|Gender=Fem|Number=Sing 3 nmod 3:nmod:для:gen _ +6 - -]DAP[ PUNCT _ Case=Acc|Number=Plur|PronType=Tot 7 punct 7:punct _ +7 длинный длинныь INTJ _ Case=Acc|Number=Plur|PronType=Tot 3 parataxis 3:parataxis SpaceAfter=No +8 , ,н PART _ Aspect=Imp|Mood=Ind|Number=Plur|Tense=Past|VerbForm=Fin|Voice=Act 9 punct 9:punct _ +9 накрытый накрыйн PART _ Aspect=Perf|Case=Nom|Gender=Masc|Number=Sing|Tense=Past|VerbForm=Part|Voice=Pass 7 conj 3:parataxis|7:conj _ +10 зеленым зелемн PROPN _ Case=Acc|Number=Plur|PronType=Tot 11 amod 11:amod _ +11 сукном суктьом PART _ Animacy=Inan|Case=Acc|Gender=Fem|Number=Sing 9 obl 9:obl:ins _ +12 и ин PART _ Case=Acc|Number=Plur|PronType=Tot 17 cc 17:cc _ +13 с о SYM _ Aspect=Imp|Mood=Ind|Number=Plur|Tense=Past|VerbForm=Fin|Voice=Act 15 case 15:case _ +14 обеих обхн SYM _ Aspect=Perf|Gender=Fem|Number=Sing|Tense=Past|Variant=Short|VerbForm=Part|Voice=Pass 15 nummod 15:nummod _ +15 сторон стонн SYM _ Animacy=Inan|Case=Acc|Gender=Fem|Number=Sing 17 obl 17:obl:с:gen _ +16 аккуратно аккуратнй ADJ _ Case=Acc|Number=Plur|PronType=Tot 17 advmod 17:advmod _ +17 заставленный заставленная ADJ _ Animacy=Inan|Case=Ins|Gender=Neut|Number=Sing 7 conj 3:parataxis|7:conj _ +18 стульями стульин PART _ Case=Acc|Number=Plur|PronType=Tot 17 obl 17:obl:ins SpaceAfter=No +19 . ть. PART _ Case=Nom|PronType=Ind 2 punct 2:punct _ + +# sent_id = 2003Anketa.xml_11 +# text = Семен Еремеевич очень не любил, когда за этот стол кто-нибудь садился, и если видел отодвинутый стул, то всегда собственноручно подвигал его на место, так чтобы спинки образовывали ровную прямую линию. +1 Семен сетьен PART _ Animacy=Inan|Case=Acc|Gender=Fem|Number=Sing 5 nsubj 5:nsubj _ +2 Еремеевич еремеевий ADJ _ Animacy=Inan|Case=Acc|Gender=Masc|Number=Sing 1 appos 1:appos _ +3 очень очень_ SYM _ Case=Acc|Degree=Pos|Gender=Fem|Number=Sing 5 advmod 5:advmod _ +4 не тьне INTJ _ Case=Ins|Degree=Pos|Gender=Fem|Number=Sing 5 advmod 5:advmod _ +5 любил любиь INTJ _ Case=Acc|Degree=Pos|Gender=Fem|Number=Sing 0 root 0:root SpaceAfter=No +6 , ,_ INTJ _ Case=Acc|Number=Plur|PronType=Tot 12 punct 12:punct _ +7 когда коан SCONJ _ Animacy=Inan|Case=Ins|Gender=Masc|Number=Sing 12 mark 12:mark _ +8 за оз SYM _ Animacy=Inan|Case=Ins|Gender=Neut|Number=Sing 10 case 10:case _ +9 этот этоа ADJ _ Animacy=Anim|Case=Nom|Gender=Masc|Number=Plur 10 det 10:det _ +10 стол стьол SYM _ Case=Nom|Degree=Pos|Number=Plur 12 obl 12:obl:за:acc _ +11 кто-нибудь кто-нибьн PROPN _ Case=Acc|Number=Plur|PronType=Tot 12 nsubj 12:nsubj _ +12 садился садиян PUNCT _ Case=Acc|Number=Plur|PronType=Tot 5 ccomp 5:ccomp SpaceAfter=No +13 , ть, PART _ Case=Acc|Number=Plur|PronType=Tot 23 punct 23:punct _ +14 и ин INTJ _ Case=Acc|Number=Plur|PronType=Tot 23 cc 23:cc _ +15 если еин PART _ Animacy=Inan|Case=Ins|Gender=Masc|Number=Sing 16 mark 16:mark _ +16 видел видеь ADJ _ Aspect=Perf|Gender=Fem|Number=Sing|Tense=Past|Variant=Short|VerbForm=Part|Voice=Pass 23 advcl 23:advcl:если _ +17 отодвинутый отодвинуйн SYM _ Case=Acc|Number=Plur|PronType=Tot 18 amod 18:amod _ +18 стул слн SYM _ Aspect=Imp|Mood=Ind|Number=Plur|Tense=Past|VerbForm=Fin|Voice=Act 16 obj 16:obj SpaceAfter=No +19 , ,н PART _ Case=Acc|Number=Plur|PronType=Tot 16 punct 23:punct _ +20 то он PUNCT _ Animacy=Inan|Case=Ins|Gender=Masc|Number=Sing 23 mark 23:mark _ +21 всегда вседсь ADJ _ Case=Acc|Number=Plur|PronType=Tot 23 advmod 23:advmod _ +22 собственноручно собственнорунсь SYM _ Case=Acc|Number=Plur|PronType=Tot 23 advmod 23:advmod _ +23 подвигал подвилн SYM _ Case=Acc|Number=Plur|PronType=Tot 5 conj 0:root|5:conj _ +24 его он SYM _ Mood=Cnd 23 obj 23:obj _ +25 на нй PROPN _ Animacy=Inan|Case=Loc|Gender=Neut|Number=Sing|PronType=Dem 26 case 26:case _ +26 место меон SYM _ Animacy=Inan|Case=Ins|Gender=Masc|Number=Sing 23 obl 23:obl:на:acc SpaceAfter=No +27 , ть, PART _ Case=Acc|Number=Plur|PronType=Tot 31 punct 31:punct _ +28 так так_ PART _ Case=Acc|Number=Plur|PronType=Tot 31 mark 31:mark _ +29 чтобы чттьбы INTJ _ Animacy=Anim|Case=Nom|Gender=Masc|Number=Plur 28 fixed 28:fixed _ +30 спинки спинок SYM _ Aspect=Imp|Mood=Ind|Number=Plur|Tense=Past|VerbForm=Fin|Voice=Act 31 nsubj 31:nsubj _ +31 образовывали образовываая [UNK] _ Case=Nom|PronType=Ind 23 advcl 23:advcl:чтобы _ +32 ровную ровюн PART _ Case=Acc|Number=Plur|PronType=Tot 34 amod 34:amod _ +33 прямую пряюн PART _ Aspect=Imp|Mood=Ind|Number=Plur|Tense=Past|VerbForm=Fin|Voice=Act 34 amod 34:amod _ +34 линию линий SYM _ Mood=Cnd 31 obj 31:obj SpaceAfter=No +35 . ть. INTJ _ Animacy=Inan|Case=Acc|Gender=Fem|Number=Sing 5 punct 5:punct _ + diff --git a/tests/testdata/datasets/ru_train.conllu b/tests/testdata/datasets/ru_train.conllu new file mode 100644 index 0000000..126982a --- /dev/null +++ b/tests/testdata/datasets/ru_train.conllu @@ -0,0 +1,255 @@ +# sent_id = 2003Anketa.xml_2 +# text = Начальник областного управления связи Семен Еремеевич был человек простой, приходил на работу всегда вовремя, здоровался с секретаршей за руку и иногда даже писал в стенгазету заметки под псевдонимом "Муха". +1 Начальник начальник NOUN _ Animacy=Anim|Case=Nom|Gender=Masc|Number=Sing 8 nsubj 8:nsubj _ +2 областного областной ADJ _ Case=Gen|Degree=Pos|Gender=Neut|Number=Sing 3 amod 3:amod _ +3 управления управление NOUN _ Animacy=Inan|Case=Gen|Gender=Neut|Number=Sing 1 nmod 1:nmod:gen _ +4 связи связь NOUN _ Animacy=Inan|Case=Gen|Gender=Fem|Number=Sing 3 nmod 3:nmod:gen _ +5 Семен Семен PROPN _ Animacy=Anim|Case=Nom|Gender=Masc|Number=Sing 1 appos 1:appos _ +6 Еремеевич Еремеевич PROPN _ Animacy=Anim|Case=Nom|Gender=Masc|Number=Sing 5 flat:name 5:flat:name _ +7 был быть AUX _ Aspect=Imp|Gender=Masc|Mood=Ind|Number=Sing|Tense=Past|VerbForm=Fin|Voice=Act 8 cop 8:cop _ +8 человек человек NOUN _ Animacy=Anim|Case=Nom|Gender=Masc|Number=Sing 0 root 0:root _ +9 простой простой ADJ _ Case=Nom|Degree=Pos|Gender=Masc|Number=Sing 8 amod 8:amod SpaceAfter=No +10 , , PUNCT _ _ 11 punct 11:punct _ +11 приходил приходить VERB _ Aspect=Imp|Gender=Masc|Mood=Ind|Number=Sing|Tense=Past|VerbForm=Fin|Voice=Act 8 conj 0:root|8:conj _ +12 на на ADP _ _ 13 case 13:case _ +13 работу работа NOUN _ Animacy=Inan|Case=Acc|Gender=Fem|Number=Sing 11 obl 11:obl:на:acc _ +14 всегда всегда ADV _ Degree=Pos 11 advmod 11:advmod _ +15 вовремя вовремя ADV _ Degree=Pos 11 advmod 11:advmod SpaceAfter=No +16 , , PUNCT _ _ 17 punct 17:punct _ +17 здоровался здороваться VERB _ Aspect=Imp|Gender=Masc|Mood=Ind|Number=Sing|Tense=Past|VerbForm=Fin|Voice=Mid 8 conj 0:root|8:conj _ +18 с с ADP _ _ 19 case 19:case _ +19 секретаршей секретарша NOUN _ Animacy=Anim|Case=Ins|Gender=Fem|Number=Sing 17 obl 17:obl:с:ins _ +20 за за ADP _ _ 21 case 21:case _ +21 руку рука NOUN _ Animacy=Inan|Case=Acc|Gender=Fem|Number=Sing 17 obl 17:obl:за:acc _ +22 и и CCONJ _ _ 25 cc 25:cc _ +23 иногда иногда ADV _ Degree=Pos 25 advmod 25:advmod _ +24 даже даже PART _ _ 25 advmod 25:advmod _ +25 писал писать VERB _ Aspect=Imp|Gender=Masc|Mood=Ind|Number=Sing|Tense=Past|VerbForm=Fin|Voice=Act 8 conj 0:root|8:conj _ +26 в в ADP _ _ 27 case 27:case _ +27 стенгазету стенгазета NOUN _ Animacy=Inan|Case=Acc|Gender=Fem|Number=Sing 25 obl 25:obl:в:acc _ +28 заметки заметка NOUN _ Animacy=Inan|Case=Acc|Gender=Fem|Number=Plur 25 obj 25:obj _ +29 под под ADP _ _ 30 case 30:case _ +30 псевдонимом псевдоним NOUN _ Animacy=Inan|Case=Ins|Gender=Masc|Number=Sing 25 obl 25:obl:под:ins _ +31 " " PUNCT _ _ 32 punct 32:punct SpaceAfter=No +32 Муха муха NOUN _ Animacy=Anim|Case=Nom|Gender=Fem|Number=Sing 30 nmod 30:nmod:nom SpaceAfter=No +33 " " PUNCT _ _ 32 punct 32:punct SpaceAfter=No +34 . . PUNCT _ _ 8 punct 8:punct _ + +# sent_id = 2003Anketa.xml_3 +# text = В приемной его с утра ожидали посетители, - кое-кто с важными делами, а кое-кто и с такими, которые легко можно было решить в нижестоящих инстанциях, не затрудняя Семена Еремеевича. +1 В в ADP _ _ 2 case 2:case _ +2 приемной приемная NOUN _ Animacy=Inan|Case=Loc|Gender=Fem|Number=Sing 6 obl 6:obl:в:loc _ +3 его он PRON _ Case=Acc|Gender=Masc|Number=Sing|Person=3|PronType=Prs 6 obj 6:obj _ +4 с с ADP _ _ 5 case 5:case _ +5 утра утро NOUN _ Animacy=Inan|Case=Gen|Gender=Neut|Number=Sing 6 obl 6:obl:с:gen _ +6 ожидали ожидать VERB _ Aspect=Imp|Mood=Ind|Number=Plur|Tense=Past|VerbForm=Fin|Voice=Act 0 root 0:root _ +7 посетители посетитель NOUN _ Animacy=Anim|Case=Nom|Gender=Masc|Number=Plur 6 nsubj 6:nsubj SpaceAfter=No +8 , , PUNCT _ _ 13 punct 13:punct _ +9 - - PUNCT _ _ 13 punct 13:punct _ +10 кое-кто кое-кто PRON _ Case=Nom|PronType=Ind 13 nsubj 13:nsubj _ +11 с с ADP _ _ 13 case 13:case _ +12 важными важный ADJ _ Case=Ins|Degree=Pos|Number=Plur 13 amod 13:amod _ +13 делами дело NOUN _ Animacy=Inan|Case=Ins|Gender=Neut|Number=Plur 7 parataxis 7:parataxis SpaceAfter=No +14 , , PUNCT _ _ 19 punct 19:punct _ +15 а а CCONJ _ _ 19 cc 19:cc _ +16 кое-кто кое-кто PRON _ Case=Nom|PronType=Ind 19 nsubj 19:nsubj _ +17 и и PART _ _ 19 advmod 19:advmod _ +18 с с ADP _ _ 19 case 19:case _ +19 такими такой DET _ Case=Ins|Number=Plur|PronType=Dem 13 det 13:det|25:obj SpaceAfter=No +20 , , PUNCT _ _ 23 punct 23:punct _ +21 которые который PRON _ Case=Acc|PronType=Int,Rel 25 obj 19:ref _ +22 легко легко ADV _ Degree=Pos 23 advmod 23:advmod _ +23 можно можно ADV _ Degree=Pos 19 acl:relcl 19:acl:relcl _ +24 было быть AUX _ Aspect=Imp|Gender=Neut|Mood=Ind|Number=Sing|Tense=Past|VerbForm=Fin|Voice=Act 23 cop 23:cop _ +25 решить решить VERB _ Aspect=Perf|VerbForm=Inf|Voice=Act 23 csubj 23:csubj _ +26 в в ADP _ _ 28 case 28:case _ +27 нижестоящих нижестоящий ADJ _ Case=Loc|Degree=Pos|Number=Plur 28 amod 28:amod _ +28 инстанциях инстанция NOUN _ Animacy=Inan|Case=Loc|Gender=Fem|Number=Plur 25 obl 25:obl:в:loc SpaceAfter=No +29 , , PUNCT _ _ 31 punct 31:punct _ +30 не не PART _ _ 31 advmod 31:advmod _ +31 затрудняя затруднять VERB _ Aspect=Imp|Tense=Pres|VerbForm=Conv|Voice=Act 25 advcl 25:advcl _ +32 Семена Семен PROPN _ Animacy=Anim|Case=Gen|Gender=Masc|Number=Sing 31 obl 31:obl:gen _ +33 Еремеевича Еремеевич PROPN _ Animacy=Anim|Case=Gen|Gender=Masc|Number=Sing 32 flat:name 32:flat:name SpaceAfter=No +34 . . PUNCT _ _ 6 punct 6:punct _ + +# sent_id = 2003Anketa.xml_4 +# text = Однако стиль работы Семена Еремеевича заключался в том, чтобы принимать всех желающих и лично вникать в дело. +1 Однако однако ADV _ Degree=Pos 6 advmod 6:advmod _ +2 стиль стиль NOUN _ Animacy=Inan|Case=Nom|Gender=Masc|Number=Sing 6 nsubj 6:nsubj _ +3 работы работа NOUN _ Animacy=Inan|Case=Gen|Gender=Fem|Number=Sing 2 nmod 2:nmod:gen _ +4 Семена Семен PROPN _ Animacy=Anim|Case=Gen|Gender=Masc|Number=Sing 3 nmod 3:nmod:gen _ +5 Еремеевича Еремеевич PROPN _ Animacy=Anim|Case=Gen|Gender=Masc|Number=Sing 4 flat:name 4:flat:name _ +6 заключался заключаться VERB _ Aspect=Imp|Gender=Masc|Mood=Ind|Number=Sing|Tense=Past|VerbForm=Fin|Voice=Mid 0 root 0:root _ +7 в в ADP _ _ 8 case 8:case _ +8 том то PRON _ Animacy=Inan|Case=Loc|Gender=Neut|Number=Sing|PronType=Dem 6 obl 6:obl:в:loc SpaceAfter=No +9 , , PUNCT _ _ 11 punct 11:punct _ +10 чтобы чтобы SCONJ _ Mood=Cnd 11 mark 11:mark _ +11 принимать принимать VERB _ Aspect=Imp|VerbForm=Inf|Voice=Act 8 advcl 8:advcl:чтобы _ +12 всех весь DET _ Case=Acc|Number=Plur|PronType=Tot 13 det 13:det _ +13 желающих желать VERB _ Animacy=Anim|Aspect=Imp|Case=Acc|Number=Plur|Tense=Pres|VerbForm=Part|Voice=Act 11 xcomp 11:xcomp _ +14 и и CCONJ _ _ 16 cc 16:cc _ +15 лично лично ADV _ Degree=Pos 16 advmod 16:advmod _ +16 вникать вникать VERB _ Aspect=Imp|VerbForm=Inf|Voice=Act 11 conj 8:advcl:чтобы|11:conj _ +17 в в ADP _ _ 18 case 18:case _ +18 дело дело NOUN _ Animacy=Inan|Case=Acc|Gender=Neut|Number=Sing 16 obl 16:obl:в:acc SpaceAfter=No +19 . . PUNCT _ _ 6 punct 6:punct _ + +# sent_id = 2003Anketa.xml_5 +# text = Приемная была обставлена просто, но по-деловому. +1 Приемная приемная NOUN _ Animacy=Inan|Case=Nom|Gender=Fem|Number=Sing 3 nsubj:pass 3:nsubj:pass _ +2 была быть AUX _ Aspect=Imp|Gender=Fem|Mood=Ind|Number=Sing|Tense=Past|VerbForm=Fin|Voice=Act 3 aux:pass 3:aux:pass _ +3 обставлена обставить VERB _ Aspect=Perf|Gender=Fem|Number=Sing|Tense=Past|Variant=Short|VerbForm=Part|Voice=Pass 0 root 0:root _ +4 просто просто ADV _ Degree=Pos 3 advmod 3:advmod SpaceAfter=No +5 , , PUNCT _ _ 7 punct 7:punct _ +6 но но CCONJ _ _ 7 cc 7:cc _ +7 по-деловому по-деловому ADV _ Degree=Pos 4 conj 3:advmod|4:conj SpaceAfter=No +8 . . PUNCT _ _ 3 punct 3:punct _ + +# sent_id = 2003Anketa.xml_6 +# text = У двери стоял стол секретарши, на столе - пишущая машинка с широкой кареткой. +1 У у ADP _ _ 2 case 2:case _ +2 двери дверь NOUN _ Animacy=Inan|Case=Gen|Gender=Fem|Number=Sing 3 obl 3:obl:у:gen _ +3 стоял стоять VERB _ Aspect=Imp|Gender=Masc|Mood=Ind|Number=Sing|Tense=Past|VerbForm=Fin|Voice=Act 0 root 0:root _ +4 стол стол NOUN _ Animacy=Inan|Case=Nom|Gender=Masc|Number=Sing 3 nsubj 3:nsubj _ +5 секретарши секретарша NOUN _ Animacy=Anim|Case=Gen|Gender=Fem|Number=Sing 4 nmod 4:nmod:gen SpaceAfter=No +6 , , PUNCT _ _ 8 punct 8:punct _ +7 на на ADP _ _ 8 case 8:case _ +8 столе стол NOUN _ Animacy=Inan|Case=Loc|Gender=Masc|Number=Sing 3 conj 0:root|3:conj _ +9 - - PUNCT _ _ 11 punct 11:punct _ +10 пишущая писать VERB _ Aspect=Imp|Case=Nom|Gender=Fem|Number=Sing|Tense=Pres|VerbForm=Part|Voice=Act 11 amod 11:amod _ +11 машинка машинка NOUN _ Animacy=Inan|Case=Nom|Gender=Fem|Number=Sing 8 nsubj 8:nsubj _ +12 с с ADP _ _ 14 case 14:case _ +13 широкой широкий ADJ _ Case=Ins|Degree=Pos|Gender=Fem|Number=Sing 14 amod 14:amod _ +14 кареткой каретка NOUN _ Animacy=Inan|Case=Ins|Gender=Fem|Number=Sing 11 nmod 11:nmod:с:ins SpaceAfter=No +15 . . PUNCT _ _ 3 punct 3:punct _ + +# sent_id = 2003Anketa.xml_7 +# text = В углу висел репродуктор и играло радио для развлечения ожидающих и еще для того, чтобы заглушать голос начальника, доносившийся из кабинета, так как, бесспорно, среди посетителей могли находиться и случайные люди. +1 В в ADP _ _ 2 case 2:case _ +2 углу угол NOUN _ Animacy=Inan|Case=Loc|Gender=Masc|Number=Sing 3 obl 3:obl:в:loc _ +3 висел висеть VERB _ Aspect=Imp|Gender=Masc|Mood=Ind|Number=Sing|Tense=Past|VerbForm=Fin|Voice=Act 0 root 0:root _ +4 репродуктор репродуктор NOUN _ Animacy=Inan|Case=Nom|Gender=Masc|Number=Sing 3 nsubj 3:nsubj _ +5 и и CCONJ _ _ 6 cc 6:cc _ +6 играло играть VERB _ Aspect=Imp|Gender=Neut|Mood=Ind|Number=Sing|Tense=Past|VerbForm=Fin|Voice=Act 3 conj 0:root|3:conj _ +7 радио радио NOUN _ Animacy=Inan|Case=Nom|Gender=Neut|Number=Sing 6 nsubj 6:nsubj _ +8 для для ADP _ _ 9 case 9:case _ +9 развлечения развлечение NOUN _ Animacy=Inan|Case=Gen|Gender=Neut|Number=Sing 6 obl 6:obl:для:gen _ +10 ожидающих ожидать VERB _ Aspect=Imp|Case=Gen|Number=Plur|Tense=Pres|VerbForm=Part|Voice=Act 9 acl 9:acl _ +11 и и CCONJ _ _ 14 cc 14:cc _ +12 еще еще ADV _ Degree=Pos 14 obl 14:obl _ +13 для для ADP _ _ 14 case 14:case _ +14 того то PRON _ Animacy=Inan|Case=Gen|Gender=Neut|Number=Sing|PronType=Dem 9 conj 6:obl:для:gen|9:conj SpaceAfter=No +15 , , PUNCT _ _ 17 punct 17:punct _ +16 чтобы чтобы SCONJ _ Mood=Cnd 17 mark 17:mark _ +17 заглушать заглушать VERB _ Aspect=Imp|VerbForm=Inf|Voice=Act 14 advcl 14:advcl:чтобы _ +18 голос голос NOUN _ Animacy=Inan|Case=Acc|Gender=Masc|Number=Sing 17 obj 17:obj _ +19 начальника начальник NOUN _ Animacy=Anim|Case=Gen|Gender=Masc|Number=Sing 18 nmod 18:nmod:gen SpaceAfter=No +20 , , PUNCT _ _ 21 punct 21:punct _ +21 доносившийся доноситься VERB _ Animacy=Inan|Aspect=Imp|Case=Acc|Gender=Masc|Number=Sing|Tense=Past|VerbForm=Part|Voice=Mid 18 acl 18:acl _ +22 из из ADP _ _ 23 case 23:case _ +23 кабинета кабинет NOUN _ Animacy=Inan|Case=Gen|Gender=Masc|Number=Sing 21 obl 21:obl:из:gen SpaceAfter=No +24 , , PUNCT _ _ 21 punct 21:punct _ +25 так так ADV _ Degree=Pos 32 mark 32:mark _ +26 как как SCONJ _ _ 25 fixed 25:fixed SpaceAfter=No +27 , , PUNCT _ _ 25 punct 25:punct _ +28 бесспорно бесспорно ADV _ Degree=Pos 32 parataxis 32:parataxis SpaceAfter=No +29 , , PUNCT _ _ 28 punct 28:punct _ +30 среди среди ADP _ _ 31 case 31:case _ +31 посетителей посетитель NOUN _ Animacy=Anim|Case=Gen|Gender=Masc|Number=Plur 33 obl 33:obl:среди:gen _ +32 могли мочь VERB _ Aspect=Imp|Mood=Ind|Number=Plur|Tense=Past|VerbForm=Fin|Voice=Act 17 advcl 17:advcl:так_как _ +33 находиться находиться VERB _ Aspect=Imp|VerbForm=Inf|Voice=Mid 32 xcomp 32:xcomp _ +34 и и PART _ _ 35 advmod 35:advmod _ +35 случайные случайный ADJ _ Case=Nom|Degree=Pos|Number=Plur 36 amod 36:amod _ +36 люди человек NOUN _ Animacy=Anim|Case=Nom|Gender=Masc|Number=Plur 32 nsubj 32:nsubj|33:nsubj SpaceAfter=No +37 . . PUNCT _ _ 3 punct 3:punct _ + +# sent_id = 2003Anketa.xml_8 +# text = Кабинет отличался скромностью, присущей Семену Еремеевичу. +1 Кабинет кабинет NOUN _ Animacy=Inan|Case=Nom|Gender=Masc|Number=Sing 2 nsubj 2:nsubj _ +2 отличался отличаться VERB _ Aspect=Imp|Gender=Masc|Mood=Ind|Number=Sing|Tense=Past|VerbForm=Fin|Voice=Mid 0 root 0:root _ +3 скромностью скромность NOUN _ Animacy=Inan|Case=Ins|Gender=Fem|Number=Sing 2 obl 2:obl:ins SpaceAfter=No +4 , , PUNCT _ _ 5 punct 5:punct _ +5 присущей присущий ADJ _ Case=Ins|Degree=Pos|Gender=Fem|Number=Sing 3 amod 3:amod _ +6 Семену Семен PROPN _ Animacy=Anim|Case=Dat|Gender=Masc|Number=Sing 5 iobj 5:iobj _ +7 Еремеевичу Еремеевич PROPN _ Animacy=Anim|Case=Dat|Gender=Masc|Number=Sing 6 flat:name 6:flat:name SpaceAfter=No +8 . . PUNCT _ _ 2 punct 2:punct _ + +# sent_id = 2003Anketa.xml_9 +# text = В глубине стоял широкий письменный стол с бронзовыми чернильницами и перед ним два кожаных кресла. +1 В в ADP _ _ 2 case 2:case _ +2 глубине глубина NOUN _ Animacy=Inan|Case=Loc|Gender=Fem|Number=Sing 3 obl 3:obl:в:loc _ +3 стоял стоять VERB _ Aspect=Imp|Gender=Masc|Mood=Ind|Number=Sing|Tense=Past|VerbForm=Fin|Voice=Act 0 root 0:root _ +4 широкий широкий ADJ _ Case=Nom|Degree=Pos|Gender=Masc|Number=Sing 6 amod 6:amod _ +5 письменный письменный ADJ _ Case=Nom|Degree=Pos|Gender=Masc|Number=Sing 6 amod 6:amod _ +6 стол стол NOUN _ Animacy=Inan|Case=Nom|Gender=Masc|Number=Sing 3 nsubj 3:nsubj _ +7 с с ADP _ _ 9 case 9:case _ +8 бронзовыми бронзовый ADJ _ Case=Ins|Degree=Pos|Number=Plur 9 amod 9:amod _ +9 чернильницами чернильница NOUN _ Animacy=Inan|Case=Ins|Gender=Fem|Number=Plur 6 nmod 6:nmod:с:ins _ +10 и и CCONJ _ _ 12 cc 12:cc _ +11 перед перед ADP _ _ 12 case 12:case _ +12 ним он PRON _ Case=Ins|Gender=Masc|Number=Sing|Person=3|PronType=Prs 3 conj 0:root|3:conj _ +13 два два NUM _ Case=Nom|Gender=Neut|NumType=Card 15 nummod:gov 15:nummod:gov _ +14 кожаных кожаный ADJ _ Case=Gen|Degree=Pos|Number=Plur 15 amod 15:amod _ +15 кресла кресло NOUN _ Animacy=Inan|Case=Gen|Gender=Neut|Number=Sing 12 nsubj 12:nsubj SpaceAfter=No +16 . . PUNCT _ _ 3 punct 3:punct _ + +# sent_id = 2003Anketa.xml_10 +# text = Справа был стол для заседаний - длинный, накрытый зеленым сукном и с обеих сторон аккуратно заставленный стульями. +1 Справа справа ADV _ Degree=Pos 2 advmod 2:advmod _ +2 был быть VERB _ Aspect=Imp|Gender=Masc|Mood=Ind|Number=Sing|Tense=Past|VerbForm=Fin|Voice=Act 0 root 0:root _ +3 стол стол NOUN _ Animacy=Inan|Case=Nom|Gender=Masc|Number=Sing 2 nsubj 2:nsubj _ +4 для для ADP _ _ 5 case 5:case _ +5 заседаний заседание NOUN _ Animacy=Inan|Case=Gen|Gender=Neut|Number=Plur 3 nmod 3:nmod:для:gen _ +6 - - PUNCT _ _ 7 punct 7:punct _ +7 длинный длинный ADJ _ Case=Nom|Degree=Pos|Gender=Masc|Number=Sing 3 parataxis 3:parataxis SpaceAfter=No +8 , , PUNCT _ _ 9 punct 9:punct _ +9 накрытый накрыть VERB _ Aspect=Perf|Case=Nom|Gender=Masc|Number=Sing|Tense=Past|VerbForm=Part|Voice=Pass 7 conj 3:parataxis|7:conj _ +10 зеленым зеленый ADJ _ Case=Ins|Degree=Pos|Gender=Neut|Number=Sing 11 amod 11:amod _ +11 сукном сукно NOUN _ Animacy=Inan|Case=Ins|Gender=Neut|Number=Sing 9 obl 9:obl:ins _ +12 и и CCONJ _ _ 17 cc 17:cc _ +13 с с ADP _ _ 15 case 15:case _ +14 обеих оба NUM _ Case=Gen|Gender=Fem|NumType=Card 15 nummod 15:nummod _ +15 сторон сторона NOUN _ Animacy=Inan|Case=Gen|Gender=Fem|Number=Plur 17 obl 17:obl:с:gen _ +16 аккуратно аккуратно ADV _ Degree=Pos 17 advmod 17:advmod _ +17 заставленный заставить VERB _ Aspect=Perf|Case=Nom|Gender=Masc|Number=Sing|Tense=Past|VerbForm=Part|Voice=Pass 7 conj 3:parataxis|7:conj _ +18 стульями стул NOUN _ Animacy=Inan|Case=Ins|Gender=Masc|Number=Plur 17 obl 17:obl:ins SpaceAfter=No +19 . . PUNCT _ _ 2 punct 2:punct _ + +# sent_id = 2003Anketa.xml_11 +# text = Семен Еремеевич очень не любил, когда за этот стол кто-нибудь садился, и если видел отодвинутый стул, то всегда собственноручно подвигал его на место, так чтобы спинки образовывали ровную прямую линию. +1 Семен Семен PROPN _ Animacy=Anim|Case=Nom|Gender=Masc|Number=Sing 5 nsubj 5:nsubj _ +2 Еремеевич Еремеевич PROPN _ Animacy=Anim|Case=Nom|Gender=Masc|Number=Sing 1 appos 1:appos _ +3 очень очень ADV _ Degree=Pos 5 advmod 5:advmod _ +4 не не PART _ _ 5 advmod 5:advmod _ +5 любил любить VERB _ Aspect=Imp|Gender=Masc|Mood=Ind|Number=Sing|Tense=Past|VerbForm=Fin|Voice=Act 0 root 0:root SpaceAfter=No +6 , , PUNCT _ _ 12 punct 12:punct _ +7 когда когда SCONJ _ _ 12 mark 12:mark _ +8 за за ADP _ _ 10 case 10:case _ +9 этот этот DET _ Case=Acc|Gender=Masc|Number=Sing|PronType=Dem 10 det 10:det _ +10 стол стол NOUN _ Animacy=Inan|Case=Acc|Gender=Masc|Number=Sing 12 obl 12:obl:за:acc _ +11 кто-нибудь кто-нибудь PRON _ Case=Nom|PronType=Ind 12 nsubj 12:nsubj _ +12 садился садиться VERB _ Aspect=Imp|Gender=Masc|Mood=Ind|Number=Sing|Tense=Past|VerbForm=Fin|Voice=Mid 5 ccomp 5:ccomp SpaceAfter=No +13 , , PUNCT _ _ 23 punct 23:punct _ +14 и и CCONJ _ _ 23 cc 23:cc _ +15 если если SCONJ _ _ 16 mark 16:mark _ +16 видел видеть VERB _ Aspect=Imp|Gender=Masc|Mood=Ind|Number=Sing|Tense=Past|VerbForm=Fin|Voice=Act 23 advcl 23:advcl:если _ +17 отодвинутый отодвинуть VERB _ Animacy=Inan|Aspect=Perf|Case=Acc|Gender=Masc|Number=Sing|Tense=Past|VerbForm=Part|Voice=Pass 18 amod 18:amod _ +18 стул стул NOUN _ Animacy=Inan|Case=Acc|Gender=Masc|Number=Sing 16 obj 16:obj SpaceAfter=No +19 , , PUNCT _ _ 16 punct 23:punct _ +20 то то SCONJ _ _ 23 mark 23:mark _ +21 всегда всегда ADV _ Degree=Pos 23 advmod 23:advmod _ +22 собственноручно собственноручно ADV _ Degree=Pos 23 advmod 23:advmod _ +23 подвигал подвигать VERB _ Aspect=Imp|Gender=Masc|Mood=Ind|Number=Sing|Tense=Past|VerbForm=Fin|Voice=Act 5 conj 0:root|5:conj _ +24 его он PRON _ Case=Acc|Gender=Masc|Number=Sing|Person=3|PronType=Prs 23 obj 23:obj _ +25 на на ADP _ _ 26 case 26:case _ +26 место место NOUN _ Animacy=Inan|Case=Acc|Gender=Neut|Number=Sing 23 obl 23:obl:на:acc SpaceAfter=No +27 , , PUNCT _ _ 31 punct 31:punct _ +28 так так ADV _ Degree=Pos 31 mark 31:mark _ +29 чтобы чтобы SCONJ _ Mood=Cnd 28 fixed 28:fixed _ +30 спинки спинка NOUN _ Animacy=Inan|Case=Nom|Gender=Fem|Number=Plur 31 nsubj 31:nsubj _ +31 образовывали образовывать VERB _ Aspect=Imp|Mood=Ind|Number=Plur|Tense=Past|VerbForm=Fin|Voice=Act 23 advcl 23:advcl:чтобы _ +32 ровную ровный ADJ _ Case=Acc|Degree=Pos|Gender=Fem|Number=Sing 34 amod 34:amod _ +33 прямую прямой ADJ _ Case=Acc|Degree=Pos|Gender=Fem|Number=Sing 34 amod 34:amod _ +34 линию линия NOUN _ Animacy=Inan|Case=Acc|Gender=Fem|Number=Sing 31 obj 31:obj SpaceAfter=No +35 . . PUNCT _ _ 5 punct 5:punct _ + diff --git a/tests/testdata/udtube_config.yaml b/tests/testdata/udtube_config.yaml new file mode 100644 index 0000000..0759b38 --- /dev/null +++ b/tests/testdata/udtube_config.yaml @@ -0,0 +1,10 @@ +checkpoint: + save_last: 'link' # Gives us a standard name. + monitor: val_loss +data: + batch_size: 5 # 2 batches per epoch. +trainer: + accelerator: cpu # Because that's what CircleCI has. + enable_progress_bar: false + max_epochs: 5 +seed_everything: 42 diff --git a/tests/udtube_test.py b/tests/udtube_test.py new file mode 100644 index 0000000..f300d8a --- /dev/null +++ b/tests/udtube_test.py @@ -0,0 +1,105 @@ +"""Full tests of training and prediction. + +This runs five epochs of training over a small toy data set, attempting to +overfit, then compares the resubstitution predictions on this set to +previously computed results. As such this is essentially a change-detector +test. Currently, English (en), Greek (el), and Russian (ru) are supported.""" + +import difflib +import os +import tempfile +import unittest + +from parameterized import parameterized + +from udtube import cli + +# Directory the unit test is located in, relative to the working directory. +DIR = os.path.relpath(os.path.dirname(__file__), os.getcwd()) +CONFIG_PATH = os.path.join(DIR, "testdata/udtube_config.yaml") +DATASETS_DIR = os.path.join(DIR, "testdata/datasets") + + +class UDTubeTest(unittest.TestCase): + def assertFileExists(self, path: str): + self.assertTrue(os.path.exists(path), msg=f"file {path} not found") + + def setUp(self): + self.tempdir = tempfile.TemporaryDirectory(prefix="udtube_test-") + self.assertFileExists(CONFIG_PATH) + + def tearDown(self): + self.tempdir.cleanup() + + @parameterized.expand( + [ + # TODO: commented out to test CircleCI test resource limitations. + # ("el", True), + ("en", True), + # TODO: ditto. + # ("ru", False), # Russian doesn't have XPOS. + ] + ) + def test_model(self, langcode: str, use_xpos: bool): + # Fits model. + train_path = os.path.join(DATASETS_DIR, f"{langcode}_train.conllu") + self.assertFileExists(train_path) + expected_path = os.path.join( + DATASETS_DIR, f"{langcode}_expected.conllu" + ) + self.assertFileExists(expected_path) + model_dir = os.path.join(self.tempdir.name, "model") + cli.udtube_python_interface( + [ + "fit", + f"--config={CONFIG_PATH}", + f"--data.model_dir={model_dir}", + f"--data.train={train_path}", + # We are trying to overfit on the training data. + f"--data.val={train_path}", + f"--model.use_xpos={use_xpos}", + ] + ) + checkpoint_path = ( + f"{self.tempdir.name}/model/lightning_logs/" + "version_0/checkpoints/last.ckpt" + ) + self.assertFileExists(checkpoint_path) + # Predicts on "expected" data. + predicted_path = os.path.join( + self.tempdir.name, f"{langcode}_predicted.conllu" + ) + cli.udtube_python_interface( + [ + "predict", + f"--config={CONFIG_PATH}", + f"--data.model_dir={model_dir}", + f"--data.predict={expected_path}", + f"--model.use_xpos={use_xpos}", + f"--prediction.path={predicted_path}", + ] + ) + self.assertFileExists(predicted_path) + diff = self._diff(predicted_path, expected_path) + self.assertEqual(diff, [], f"Differences found:\n{diff}") + + @staticmethod + def _diff(predicted_path: str, expected_path: str) -> list: + with ( + open(predicted_path, "r") as predicted, + open(expected_path, "r") as expected, + ): + diff = list( + difflib.unified_diff( + predicted.readlines(), + expected.readlines(), + fromfile=predicted_path, + tofile=expected_path, + n=1, + ) + ) + return diff + + +if __name__ == "__main__": + unittest.main() diff --git a/udtube/callbacks.py b/udtube/callbacks.py index d06540a..969b7e5 100644 --- a/udtube/callbacks.py +++ b/udtube/callbacks.py @@ -34,8 +34,8 @@ def __init__( self.mapper = data.Mapper.read(model_dir) def __del__(self): - # This appears to be harmless on sys.stdout. - self.sink.close() + if self.sink is not sys.stdout: + self.sink.close() # Required API. diff --git a/udtube/cli.py b/udtube/cli.py index 950d87c..39235a7 100644 --- a/udtube/cli.py +++ b/udtube/cli.py @@ -7,6 +7,17 @@ from . import callbacks, data, models, trainers +def udtube_python_interface(args: cli.ArgsType = None): + """Interface to use models through Python.""" + UDTubeCLI( + models.UDTube, + data.DataModule, + auto_configure_optimizers=False, + trainer_class=trainers.Trainer, + args=args, + ) + + class UDTubeCLI(cli.LightningCLI): """The UDTube CLI interface.