diff --git a/environment.yml b/environment.yml new file mode 100644 index 0000000..417ecaa --- /dev/null +++ b/environment.yml @@ -0,0 +1,8 @@ +name: irit-rst-dt +dependencies: + - python=2.7 + - graphviz=2.38.0 + - nltk + - scikit-learn + - pip: + - "--editable=git+https://github.com/nlhepler/pydot.git#egg=pydot" diff --git a/evals/__init__.py b/evals/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/evals/attelo_predictions_to_disdep.py b/evals/attelo_predictions_to_disdep.py new file mode 100755 index 0000000..2c5a6c3 --- /dev/null +++ b/evals/attelo_predictions_to_disdep.py @@ -0,0 +1,114 @@ +"""Small utility script to convert predictions from attelo to dis_dep files. +""" + +from __future__ import absolute_import, print_function + +from collections import defaultdict +from glob import glob +import os + +from attelo.io import load_edus, load_predictions +from attelo.metrics.util import barebones_rst_deptree +from attelo.table import UNRELATED +from educe.corpus import FileId +from educe.learning.disdep_format import dump_disdep_files +from educe.rst_dt.dep2con import (DummyNuclearityClassifier, + InsideOutAttachmentRanker) + + +def attelo_predictions_to_disdep_files(edus_file_glob, edges_file, out_dir, + nary_enc_pred='tree'): + """Generate disdep files from a file dump of attelo predictions. + + Parameters + ---------- + edus_file_glob: str + Regex for `edu_input` file paths. + edges_file: str + Path to the file that contains attelo predictions (edges as + triples). + out_dir: str + Path to the output folder. + nary_enc_pred: one of {'chain', 'tree'} + Encoding for n-ary cnodes in the predicted dtree ; here it + currently triggers the strictness of the order assumed by the + dtree postprocessor: nary_enc_pred='chain' implies order='strict', + nary_enc_pred='tree' implies order='weak'. + """ + order = 'weak' if nary_enc_pred == 'tree' else 'strict' + # set up heuristic classifiers for nuclearity and rank + nuc_clf = DummyNuclearityClassifier(strategy='unamb_else_most_frequent') + nuc_clf.fit([], []) # dummy fit + rnk_clf = InsideOutAttachmentRanker(strategy='sdist-edist-rl', + prioritize_same_unit=True, + order=order) + rnk_clf.fit([], []) # dummy fit + + # load EDUs + doc_edus = dict() + id2doc = dict() + for edu_input_file in glob(edus_file_glob): + doc_name = os.path.basename(edu_input_file).rsplit('.', 4)[0] # FRAGILE + edus = load_edus(edu_input_file) + assert doc_name == edus[0].grouping + # map doc_name to list of EDUs ; populate reverse mapping from + # EDU id to doc_name, so that we can dispatch edges to their + # document + # we keep the list of EDUs sorted as in edu_input, hence we + # assume edu_input follows the linear order of EDUs + doc_edus[doc_name] = edus + for edu in edus: + id2doc[edu.id] = doc_name + # load edges and dispatch them to their doc + edges_pred = load_predictions(edges_file) + # for each doc, list edges + doc_edges = defaultdict(list) + for gov_id, dep_id, lbl in edges_pred: + if lbl != UNRELATED: + doc_name = id2doc[dep_id] + doc_edges[doc_name].append((gov_id, dep_id, lbl)) + + # for each doc, get a full-fledged RstDepTree, nuclearity and ranking + # are currently determined heuristically + doc_dtree = dict() + for doc_name, edus in doc_edus.items(): + # comply with current API for barebones_rst_deptree: + # for each doc, create a dict with one item (doc_name, list of edges) + dep_edges = doc_edges[doc_name] + # create a barebones RST dep tree: head and label only + dtree, edu2sent = barebones_rst_deptree(dep_edges, edus, strict=False) + # set its origin + dtree.origin = FileId(doc_name, None, None, None) + # flesh out with heuristically-determined nuclearity + dtree.nucs = nuc_clf.predict([dtree])[0] + # and heuristically-determined rank (needs edu2sent to prioritize + # intra-sentential attachments over inter-sentential ones) + dtree.sent_idx = edu2sent # DIRTY + dtree.ranks = rnk_clf.predict([dtree])[0] + doc_dtree[doc_name] = dtree + + # write the disdep files + dump_disdep_files(doc_dtree.values(), out_dir) + + +if __name__ == '__main__': + nary_enc_pred = 'tree' + edus_file_glob = os.path.join('TMP', 'latest', 'data', 'TEST', + '*.edu-pairs.sparse.edu_input') + edges_file_glob = os.path.join( + 'TMP', 'latest', 'scratch-current', + 'combined', + # 'output.*' + 'output.maxent-iheads-global-AD.L-jnt-eisner' + ) + # attelo predictions are currently stored in one big file + edges_files = glob(edges_file_glob) + assert len(edges_files) == 1 + edges_file = edges_files[0] + # paths to the resulting disdep files + out_dir = os.path.join('TMP_disdep', nary_enc_pred, 'ours', 'test') + if not os.path.exists(out_dir): + os.makedirs(out_dir) + # do the conversion + attelo_predictions_to_disdep_files(edus_file_glob, edges_file, out_dir, + nary_enc_pred=nary_enc_pred) diff --git a/evals/braud_coling.py b/evals/braud_coling.py new file mode 100644 index 0000000..4856aac --- /dev/null +++ b/evals/braud_coling.py @@ -0,0 +1,160 @@ +"""Read the output of Braud et al.'s COLING parser. + +""" + +from __future__ import absolute_import, print_function + +import codecs +from glob import glob +import itertools +import os + +from nltk import Tree + +from educe.annotation import Span +from educe.rst_dt.annotation import EDU, Node, SimpleRSTTree +from educe.rst_dt.deptree import RstDepTree + + +# map *.mrg.pred files to the original doc names +MRG_TO_RST = { + '12.mrg.pred': 'wsj_0644.out', # 4 + '4.mrg.pred': 'wsj_1129.out', # 5 + '26.mrg.pred': 'wsj_1197.out', # 6 + '24.mrg.pred': 'wsj_1113.out', # 8 + '14.mrg.pred': 'wsj_0684.out', # 10 + '32.mrg.pred': 'wsj_1354.out', # 11 + '18.mrg.pred': 'wsj_1183.out', # 12 + '29.mrg.pred': 'wsj_1346.out', # 15 + '28.mrg.pred': 'wsj_1169.out', # 17 + '37.mrg.pred': 'wsj_0667.out', # 17 + '19.mrg.pred': 'wsj_0607.out', # 19 + '7.mrg.pred': 'wsj_0654.out', # 19 + '16.mrg.pred': 'wsj_1325.out', # 21 + '25.mrg.pred': 'wsj_2375.out', # 22 + '31.mrg.pred': 'wsj_1380.out', # 23 + '1.mrg.pred': 'wsj_0623.out', # 25 + '15.mrg.pred': 'wsj_2373.out', # 31 + '30.mrg.pred': 'wsj_2336.out', # 31 + '3.mrg.pred': 'wsj_1365.out', # 39 + '34.mrg.pred': 'wsj_1148.out', # 43 + '11.mrg.pred': 'wsj_1306.out', # 47 + '10.mrg.pred': 'wsj_2354.out', # 52 + '35.mrg.pred': 'wsj_1126.out', # 55 + '0.mrg.pred': 'wsj_2385.out', # 60 + '2.mrg.pred': 'wsj_0632.out', # 62 + '20.mrg.pred': 'wsj_0602.out', # 69 + '27.mrg.pred': 'wsj_0627.out', # 69 + '13.mrg.pred': 'wsj_1189.out', # 91 + '6.mrg.pred': 'wsj_0616.out', # 92 + '36.mrg.pred': 'wsj_1307.out', # 98 + '33.mrg.pred': 'wsj_1142.out', # 106 + '9.mrg.pred': 'wsj_0655.out', # 110 + '21.mrg.pred': 'wsj_2386.out', # 127 + '23.mrg.pred': 'wsj_0689.out', # 132 + '8.mrg.pred': 'wsj_1387.out', # 134 + '17.mrg.pred': 'wsj_1331.out', # 158 + '22.mrg.pred': 'wsj_1376.out', # 202 + '5.mrg.pred': 'wsj_1146.out', # 304 +} + + +def tree_to_simple_rsttree(tree): + """Build a SimpleRSTTree from a NLTK Tree""" + origin = None # or is it? + if not tree: + # no kid: EDU (+pre-terminal) + num = int(tree.label()) + span = Span(num, num) # FIXME + text = '' # FIXME + edu = EDU(num, span, text, context=None, origin=origin) + # pre-terminal + edu_span = (num, num) + nuc = "leaf" + rel = "leaf" + node = Node(nuc, edu_span, span, rel, context=None) + return SimpleRSTTree(node, [edu], origin=origin) + + # internal node + new_kids = [tree_to_simple_rsttree(kid) for kid in tree] + # node + nuc, rel = tree.label().split('-', 1) + # map to our coarse rel names + if rel == 'Textual-organization': + rel = 'Textual' + # end map + edu_beg = (new_kids[0].num if isinstance(new_kids[0], EDU) + else new_kids[0].label().edu_span[0]) + edu_end = (new_kids[-1].num if isinstance(new_kids[-1], EDU) + else new_kids[-1].label().edu_span[1]) + edu_span = (edu_beg, edu_end) + char_beg = (new_kids[0].num if isinstance(new_kids[0], EDU) + else new_kids[0].label().span.char_start) + char_end = (new_kids[-1].num if isinstance(new_kids[-1], EDU) + else new_kids[-1].label().span.char_end) + span = Span(char_beg, char_end) + new_node = Node(nuc, edu_span, span, rel, context=None) + new_tree = SimpleRSTTree(new_node, new_kids, origin=origin) + return new_tree + + +def _load_braud_coling_file(f): + """Do load file""" + tree = Tree.fromstring(f.read().strip()) + simple_ctree = tree_to_simple_rsttree(tree) + return simple_ctree + + +def load_braud_coling_file(fpath): + """Load a file.""" + with codecs.open(fpath, 'rb', 'utf-8') as f: + return _load_braud_coling_file(f) + + +def load_braud_coling_ctrees(out_dir, rel_conv): + """Load the ctrees output by Braud et al.'s parser + + Parameters + ---------- + out_dir : str + Path to the output directory. + + rel_conv : TODO + Relation converter + + Returns + ------- + ctree_pred : dict(str, RSTTree) + RST c-tree for each document. + """ + ctree_pred = dict() + for fpath in sorted(glob(os.path.join(out_dir, '*.mrg.pred'))): + fname = os.path.basename(fpath) + doc_name = MRG_TO_RST.get(fname, fname) + sct_pred = load_braud_coling_file(fpath) + # convert to regular RSTTree + ct_pred = SimpleRSTTree.to_binary_rst_tree(sct_pred) + # convert relation labels + ct_pred = rel_conv(ct_pred) + # TODO check ct_true: assert that mrg.gold == .out.dis + ctree_pred[doc_name] = ct_pred + return ctree_pred + + +def load_braud_coling_dtrees(out_dir, rel_conv, nary_enc='chain', + ctree_pred=None): + """Do load dtrees. + + Parameters + ---------- + ctree_pred : dict(str, RSTTree), optional + RST c-trees, indexed by doc_name. If c-trees are provided this + way, `out_dir` is ignored. + """ + dtree_pred = dict() + if ctree_pred is None: + ctree_pred = load_braud_coling_ctrees(out_dir, rel_conv) + for doc_name, ct_pred in ctree_pred.items(): + dt_pred = RstDepTree.from_rst_tree(ct_pred) + dtree_pred[doc_name] = dt_pred + return dtree_pred diff --git a/evals/braud_eacl.py b/evals/braud_eacl.py new file mode 100644 index 0000000..e865a8e --- /dev/null +++ b/evals/braud_eacl.py @@ -0,0 +1,141 @@ +"""Read the output of Braud et al.'s EACL parsers. + +""" + +from __future__ import absolute_import, print_function + +import codecs +import itertools +from glob import glob +import os + +from nltk import Tree + +from educe.annotation import Span +from educe.rst_dt.annotation import EDU, Node, SimpleRSTTree +from educe.rst_dt.deptree import RstDepTree + + +def tree_to_simple_rsttree(tree, edu_num=1): + """Build a SimpleRSTTree from a NLTK Tree. + + Parameters + ---------- + edu_num : int, defaults to 1 + Number of the next EDU + """ + origin = None + + if tree.label() == 'EDU': + # EDU (+pre-terminal) + num = edu_num + span = Span(num, num) + # 'EDU ' + text = tree[0] + edu = EDU(num, span, text, context=None, origin=origin) + # pre-terminal + edu_span = (num, num) + nuc = "leaf" + rel = "leaf" + node = Node(nuc, edu_span, span, rel, context=None) + return SimpleRSTTree(node, [edu], origin=origin) + + new_kids = [] + for kid in tree: + new_kid = tree_to_simple_rsttree(kid, edu_num=edu_num) + edu_num = new_kid.label().edu_span[1] + 1 + new_kids.append(new_kid) + + # ROOT + if tree.label() == 'ROOT': + assert len(new_kids) == 1 + return new_kids[0] + + # internal node + # label: 'NNTextualorganization' + nuc = tree.label()[:2] + rel = tree.label()[2:] + # map to our coarse rel names + rel_map = { + 'MannerMeans': 'manner-means', + 'Sameunit': 'same-unit', + 'TopicChange': 'topic-change', + 'TopicComment': 'topic-comment', + } + rel = rel_map.get(rel, rel) + # end map + + # same as in braud_coling + edu_beg = (new_kids[0].num if isinstance(new_kids[0], EDU) + else new_kids[0].label().edu_span[0]) + edu_end = (new_kids[-1].num if isinstance(new_kids[-1], EDU) + else new_kids[-1].label().edu_span[1]) + edu_span = (edu_beg, edu_end) + char_beg = (new_kids[0].num if isinstance(new_kids[0], EDU) + else new_kids[0].label().span.char_start) + char_end = (new_kids[-1].num if isinstance(new_kids[-1], EDU) + else new_kids[-1].label().span.char_end) + span = Span(char_beg, char_end) + new_node = Node(nuc, edu_span, span, rel, context=None) + new_tree = SimpleRSTTree(new_node, new_kids, origin=origin) + return new_tree + + +def _load_braud_eacl_file(f): + """Do load SimpleRSTTrees from f""" + sctrees = [] + for line in f: + tree = Tree.fromstring(line.strip()) + sctree = tree_to_simple_rsttree(tree) + sctrees.append(sctree) + return sctrees + + +def load_braud_eacl_file(fpath): + """Load SimpleRSTTrees from a file""" + with codecs.open(fpath, 'rb', 'utf-8') as f: + return _load_braud_eacl_file(f) + + +def load_braud_eacl_ctrees(fpath, rel_conv, doc_names): + """Load the ctrees output by Braud et al.'s parser + + Parameters + ---------- + fpath : str + Path to the output file. + + rel_conv : TODO + Relation converter. + + Returns + ------- + ctree_pred : dict(str, RSTTree) + RST c-tree for each document. + """ + ctree_pred = dict() + sctree_pred = load_braud_eacl_file(fpath) + for doc_name, sct_pred in zip(doc_names, sctree_pred): + ct_pred = SimpleRSTTree.to_binary_rst_tree(sct_pred) + ct_pred = rel_conv(ct_pred) + ctree_pred[doc_name] = ct_pred + return ctree_pred + + +def load_braud_eacl_dtrees(fpath, rel_conv, doc_names, nary_enc='chain', + ctree_pred=None): + """Do load dtrees + + Parameters + ---------- + ctree_pred : dict(str, RSTTree), optional + RST c-trees, indexed by doc_name. If c-trees are provided this + way, `out_dir` is ignored. + """ + dtree_pred = dict() + if ctree_pred is None: + ctree_pred = load_braud_eacl_ctrees(fpath, rel_conv, doc_names) + for doc_name, ct_pred in ctree_pred.items(): + dt_pred = RstDepTree.from_rst_tree(ct_pred) + dtree_pred[doc_name] = dt_pred + return dtree_pred diff --git a/evals/codra.py b/evals/codra.py new file mode 100644 index 0000000..11b5aea --- /dev/null +++ b/evals/codra.py @@ -0,0 +1,193 @@ +"""This module enables to load the output of Joty's discourse parser CODRA. + +""" + +from __future__ import absolute_import, print_function + +import codecs +from collections import defaultdict +import glob +import itertools +import os + +from educe.rst_dt.deptree import RstDepTree +from educe.rst_dt.parse import parse_rst_dt_tree + + +def load_codra_output_files(container_path, level='doc'): + """Load ctrees output by CODRA on the TEST section of RST-WSJ. + + Parameters + ---------- + container_path: string + Path to the main folder containing CODRA's output + + level: {'doc', 'sent'}, optional (default='doc') + Level of decoding: document-level or sentence-level + + Returns + ------- + data: dict + Dictionary that should be akin to a sklearn Bunch, with + interesting keys 'filenames', 'doc_names' and 'rst_ctrees'. + + Notes + ----- + To ensure compatibility with the rest of the code base, doc_names + are automatically added the ".out" extension. This would not work + for fileX documents, but they are absent from the TEST section of + the RST-WSJ treebank. + """ + if level == 'doc': + file_ext = '.doc_dis' + elif level == 'sent': + file_ext = '.sen_dis' + else: + raise ValueError("level {} not in ['doc', 'sent']".format(level)) + + # find all files with the right extension + pathname = os.path.join(container_path, '*{}'.format(file_ext)) + # filenames are sorted by name to avoid having to realign data + # loaded with different functions + filenames = sorted(glob.glob(pathname)) # glob.glob() returns a list + + # find corresponding doc names + doc_names = [os.path.splitext(os.path.basename(filename))[0] + '.out' + for filename in filenames] + + # load the RST trees + rst_ctrees = [] + for filename in filenames: + with codecs.open(filename, 'r', 'utf-8') as f: + # TODO (?) add support for and use RSTContext + rst_ctree = parse_rst_dt_tree(f.read(), None) + rst_ctrees.append(rst_ctree) + + data = dict(filenames=filenames, + doc_names=doc_names, + rst_ctrees=rst_ctrees) + + return data + + +def load_codra_ctrees(codra_out_dir, rel_conv): + """Load the ctrees output by CODRA as .dis files. + + This currently runs on the document-level files (.doc_dis). + + Parameters + ---------- + codra_out_dir: str + Path to the base directory containing the output files. + + Returns + ------- + ctree_pred: dict(str, RSTTree) + RST ctree for each document. + """ + # load predicted trees + data_pred = load_codra_output_files(codra_out_dir) + # filenames = data_pred['filenames'] + doc_names_pred = data_pred['doc_names'] + rst_ctrees_pred = data_pred['rst_ctrees'] + + # build a dict from doc_name to ctree (RSTTree) + ctree_pred = dict() # constituency trees + for doc_name, ct_pred in itertools.izip(doc_names_pred, rst_ctrees_pred): + # constituency tree + # replace fine-grained labels with coarse-grained labels ; + # the files we have already contain the coarse labels, except their + # initial letter is capitalized whereas ours are not + if rel_conv is not None: + ct_pred = rel_conv(ct_pred) + ctree_pred[doc_name] = ct_pred + + return ctree_pred + + +def load_codra_dtrees(codra_out_dir, rel_conv, nary_enc='chain', + ctree_pred=None): + """Get the dtrees that correspond to the ctrees output by CODRA. + + Parameters + ---------- + codra_out_dir: str + Path to the base directory containing the output files. + nary_enc: one of {'chain', 'tree'} + Encoding for n-ary nodes. + ctree_pred : dict(str, RSTTree), optional + RST c-trees, indexed by doc_name. If c-trees are provided this + way, `out_dir` is ignored. + + Returns + ------- + dtree_pred: dict(str, RstDepTree) + RST dtree for each document. + """ + if ctree_pred is None: + # load predicted trees + data_pred = load_codra_output_files(codra_out_dir) + # filenames = data_pred['filenames'] + doc_names_pred = data_pred['doc_names'] + rst_ctrees_pred = data_pred['rst_ctrees'] + ctree_pred = {doc_name: ct_pred for doc_name, ct_pred + in itertools.izip(doc_names_pred, rst_ctrees_pred)} + # build a dict from doc_name to ordered dtree (RstDepTree) + dtree_pred = dict() + for doc_name, ct_pred in ctree_pred.items(): + # constituency tree + # replace fine-grained labels with coarse-grained labels ; + # the files we have already contain the coarse labels, except their + # initial letter is capitalized whereas ours are not + if rel_conv is not None: + ct_pred = rel_conv(ct_pred) + # convert to an ordered dependency tree ; + # * 'tree' produces a weakly-ordered dtree strictly equivalent + # to the original ctree, + # * 'chain' produces a strictly-ordered dtree for which strict + # equivalence is not preserved + dt_pred = RstDepTree.from_rst_tree(ct_pred, nary_enc=nary_enc) + dtree_pred[doc_name] = dt_pred + + return dtree_pred + + +# TODO move this generic util to a more appropriate place. +# This implementation is quite ad-hoc, tailored for RST e.g. to retrieve +# the edu_num, so I would need to generalize this code first. +def get_edu2sent(att_edus): + """Get edu2sent mapping, from a list of attelo EDUs. + + Parameters + ---------- + att_edus: list of attelo EDUs + List of attelo EDUs, as produced by `load_edus`. + + Returns + ------- + doc_name2edu2sent: dict(str, [int]) + For each document, get the sentence index for every EDU. + + Example: + ``` + att_edus = load_edus(edus_file) + doc_name2edu2sent = get_edu2sent(att_edus) + for doc_name, edu2sent in doc_name2edu2sent.items(): + dtree[doc_name].edu2sent = edu2sent + ``` + + """ + edu2sent_idx = defaultdict(dict) + for att_edu in att_edus: + doc_name = att_edu.grouping + edu_num = int(att_edu.id.rsplit('_', 1)[1]) + sent_idx = int(att_edu.subgrouping.split('_sent')[1]) + edu2sent_idx[doc_name][edu_num] = sent_idx + # sort EDUs by num + # rebuild educe-style edu2sent ; prepend 0 for the fake root + doc_name2edu2sent = { + doc_name: ([0] + + [s_idx for e_num, s_idx in sorted(edu2sent.items())]) + for doc_name, edu2sent in edu2sent_idx.items() + } + return doc_name2edu2sent diff --git a/evals/dis2disdep.py b/evals/dis2disdep.py new file mode 100755 index 0000000..5825cfc --- /dev/null +++ b/evals/dis2disdep.py @@ -0,0 +1,176 @@ +"""Convert RST trees to their dependency version (.dis to .dis_dep). + +TODO +---- +* [ ] support intra-sentential level document parsing ; required to score + Joty's .sen_dis files + +""" +from __future__ import absolute_import, print_function +import argparse +import os + +from educe.corpus import FileId +from educe.learning.disdep_format import dump_disdep_files +from educe.rst_dt.corpus import Reader, RstRelationConverter +from educe.rst_dt.deptree import RstDepTree +from educe.rst_dt.feng import load_feng_output_files +from educe.rst_dt.rst_wsj_corpus import (DOUBLE_FOLDER, TEST_FOLDER, + TRAIN_FOLDER) + +from evals.codra import load_codra_output_files +from evals.gcrf_tree_format import load_gcrf_dtrees +from evals.hayashi_cons import load_hayashi_hilda_dtrees +from evals.hayashi_deps import load_hayashi_dep_dtrees +from evals.ji import load_ji_dtrees +from evals.showdown import (setup_dtree_postprocessor, NUC_STRATEGY, + NUC_CONSTANT, RNK_STRATEGY, RNK_PRIORITY_SU) + + +# original RST corpus +RST_CORPUS = os.path.join('/home/mmorey/corpora/rst_discourse_treebank/data') +RST_MAIN_TRAIN = os.path.join(RST_CORPUS, TRAIN_FOLDER) +RST_MAIN_TEST = os.path.join(RST_CORPUS, TEST_FOLDER) +RST_DOUBLE = os.path.join(RST_CORPUS, DOUBLE_FOLDER) + +# get edu2sent, set up rnk_clf and nuc_clf to predict rank and order for +# the output of Hayashi's MST parser +# * new style .edu_input: one file per doc in test set +EDUS_FILE_PAT = "TMP/latest/data/TEST/{}.relations.edu-pairs.sparse.edu_input" + +# relation converter (fine- to coarse-grained labels) +RELMAP_FILE = os.path.join('/home/mmorey/melodi/educe', + 'educe', 'rst_dt', + 'rst_112to18.txt') +REL_CONV_BASE = RstRelationConverter(RELMAP_FILE) +REL_CONV = REL_CONV_BASE.convert_tree +REL_CONV_DTREE = REL_CONV_BASE.convert_dtree +# output of Joty's parser +OUT_JOTY = os.path.join('/home/mmorey/melodi/rst/joty/Doc-level/') +# output of Feng & Hirst's parsers +FENG_BASEDIR = '/home/mmorey/melodi/rst/feng_hirst' +OUT_FENG = os.path.join(FENG_BASEDIR, 'phil/tmp/') +OUT_FENG2 = os.path.join(FENG_BASEDIR, + 'gCRF_dist/texts/results/test_batch_gold_seg') +# output of Ji's parser +JI_BASEDIR = '/home/mmorey/melodi/rst/ji_eisenstein' +OUT_JI = os.path.join(JI_BASEDIR, 'DPLP/data/docs/test/') +# output of Hayashi et al.'s parsers +HAYASHI_BASEDIR = '/home/mmorey/melodi/rst/hayashi/SIGDIAL/' +OUT_HAYASHI_MST = os.path.join(HAYASHI_BASEDIR, 'auto_parse/dep/li/') +OUT_HAYASHI_HILDA = os.path.join(HAYASHI_BASEDIR, 'auto_parse/cons/trans_li/') + + +def main(): + """Main""" + parser = argparse.ArgumentParser( + description='Convert .dis files to .dis_dep' + ) + parser.add_argument('--nary_enc', default='chain', + choices=['chain', 'tree'], + help="Encoding for n-ary nodes") + parser.add_argument('--author', default='gold', + choices=['gold', 'silver', + 'joty', 'feng', 'feng2', 'ji', + 'hayashi_hilda', 'hayashi_mst'], + help="Author of the version of the corpus") + parser.add_argument('--split', default='test', + choices=['train', 'test', 'double'], + help="Relevant part of the corpus") + parser.add_argument('--out_root', default='TMP_disdep', + help="Root directory for the output") + args = parser.parse_args() + # precise output path, by default: TMP_disdep/chain/gold/train + out_dir = os.path.join(args.out_root, args.nary_enc, args.author, + args.split) + if not os.path.exists(out_dir): + os.makedirs(out_dir) + # read RST trees + nary_enc = args.nary_enc + author = args.author + corpus_split = args.split + + if author == 'gold': + if corpus_split == 'train': + corpus_dir = RST_MAIN_TRAIN + elif corpus_split == 'test': + corpus_dir = RST_MAIN_TEST + elif corpus_split == 'double': + raise NotImplementedError("Gold trees for 'double'") + reader = Reader(corpus_dir) + rtrees = reader.slurp() + dtrees = {doc_name: RstDepTree.from_rst_tree(rtree, nary_enc=nary_enc) + for doc_name, rtree in rtrees.items()} + elif author == 'silver': + if corpus_split == 'double': + corpus_dir = RST_DOUBLE + else: + raise ValueError("'silver' annotation is available for the " + "'double' split only") + elif author == 'joty': + if corpus_split != 'test': + raise ValueError("The output of Joty's parser is available for " + "the 'test' split only") + data_pred = load_codra_output_files(OUT_JOTY, level='doc') + doc_names = data_pred['doc_names'] + rtrees = data_pred['rst_ctrees'] + dtrees = {doc_name: RstDepTree.from_rst_tree(rtree, nary_enc=nary_enc) + for doc_name, rtree in zip(doc_names, rtrees)} + # set reference to the document in the RstDepTree (required by + # dump_disdep_files) + for doc_name, dtree in dtrees.items(): + dtree.origin = FileId(doc_name, None, None, None) + elif author == 'feng': + if corpus_split != 'test': + raise ValueError("The output of Feng & Hirst's parser is " + "available for the 'test' split only") + data_pred = load_feng_output_files(OUT_FENG) + doc_names = data_pred['doc_names'] + rtrees = data_pred['rst_ctrees'] + dtrees = {doc_name: RstDepTree.from_rst_tree(rtree, nary_enc=nary_enc) + for doc_name, rtree in zip(doc_names, rtrees)} + # set reference to the document in the RstDepTree (required by + # dump_disdep_files) + for doc_name, dtree in dtrees.items(): + dtree.origin = FileId(doc_name, None, None, None) + + elif author == 'feng2': + if corpus_split != 'test': + raise ValueError("The output of Feng & Hirst's parser is " + "available for the 'test' split only") + dtrees = load_gcrf_dtrees(OUT_FENG2, REL_CONV) + for doc_name, dtree in dtrees.items(): + dtree.origin = FileId(doc_name, None, None, None) + + elif author == 'ji': + if corpus_split != 'test': + raise ValueError("The output of Ji & Eisenstein's parser is " + "available for the 'test' split only") + dtrees = load_ji_dtrees(OUT_JI, REL_CONV) + elif author == 'hayashi_mst': + if corpus_split != 'test': + raise ValueError("The output of Hayashi et al.'s parser is " + "available for the 'test' split only") + # setup nuc_clf, rnk_clf + nuc_clf, rnk_clf = setup_dtree_postprocessor( + nary_enc='tree', order='weak', + nuc_strategy=NUC_STRATEGY, + nuc_constant=NUC_CONSTANT, + rnk_strategy=RNK_STRATEGY, + rnk_prioritize_same_unit=RNK_PRIORITY_SU) + # end setup + dtrees = load_hayashi_dep_dtrees( + OUT_HAYASHI_MST, REL_CONV_DTREE, EDUS_FILE_PAT, + nuc_clf, rnk_clf) + elif author == 'hayashi_hilda': + if corpus_split != 'test': + raise ValueError("The output of Hayashi et al.'s parser is " + "available for the 'test' split only") + dtrees = load_hayashi_hilda_dtrees(OUT_HAYASHI_HILDA, REL_CONV) + + # do dump + dump_disdep_files(dtrees.values(), out_dir) + + +if __name__ == '__main__': + main() diff --git a/evals/eval_disdep.py b/evals/eval_disdep.py new file mode 100755 index 0000000..7f84965 --- /dev/null +++ b/evals/eval_disdep.py @@ -0,0 +1,116 @@ +"""Evaluation procedure for discourse dependency (disdep) files. + +Computes UAS and flavours of LAS for labels, nuclearity, rank and +their combinations. +""" + +from __future__ import absolute_import, print_function +import argparse +import codecs +import csv +from glob import glob +import os + + +if __name__ == '__main__': + parser = argparse.ArgumentParser( + description="Evaluate dis_dep trees against a given reference") + parser.add_argument('authors_pred', nargs='+', + choices=['gold', 'silver', + 'joty', 'feng', 'feng2', 'ji', + 'hayashi_hilda', 'hayashi_mst', + 'ours'], + help="Author(s) of the predictions") + parser.add_argument('--author_true', default='gold', + choices=['gold', 'silver', + 'joty', 'feng', 'feng2', 'ji', + 'hayashi_hilda', 'hayashi_mst', + 'ours'], + help="Author of the reference") + parser.add_argument('--nary_enc', default='chain', + choices=['tree', 'chain'], + help="Encoding of n-ary nodes") + # TODO add argparse param for split + args = parser.parse_args() + author_true = args.author_true + authors_pred = args.authors_pred + nary_enc = args.nary_enc + # reference + dir_true = os.path.join('TMP_disdep', nary_enc, author_true, 'test') + files_true = {os.path.basename(f).rsplit('.')[0]: f + for f in glob(os.path.join(dir_true, '*.dis_dep'))} + # table header + len_author_str = max(len(x) for x in authors_pred) + print('\t'.join([ + '{parser_name: <{width}}'.format( + parser_name='parser', width=len_author_str), + 'a', 'l', 'n', 'r', + 'al', 'an', 'ar', + 'aln', 'alr', + 'alnr', + 'support' + ])) + + for author_pred in authors_pred: + dir_pred = os.path.join('TMP_disdep', nary_enc, author_pred, 'test') + files_pred = {os.path.basename(f).rsplit('.')[0]: f + for f in glob(os.path.join(dir_pred, '*.dis_dep'))} + assert sorted(files_true.keys()) == sorted(files_pred.keys()) + + cnt_tot = 0 # total deps + cnt_a = 0 # correct heads (attachments) + cnt_l = 0 # correct labels + cnt_n = 0 # correct nuclearity + cnt_r = 0 # correct ranks + cnt_al = 0 # correct labelled attachments + cnt_an = 0 # correct attachment + nuc + cnt_ar = 0 # correct attachment + rank + cnt_aln = 0 # correct attachment + label + nuc + cnt_alr = 0 # correct attachment + label + rank + cnt_alnr = 0 # correct attachment + label + nuc + rank + + for doc_name, f_true in files_true.items(): + f_pred = files_pred[doc_name] + with codecs.open(f_true, 'r', encoding='utf-8') as f_true: + with codecs.open(f_pred, 'r', encoding='utf-8') as f_pred: + reader_true = csv.reader(f_true, dialect=csv.excel_tab) + reader_pred = csv.reader(f_pred, dialect=csv.excel_tab) + for line_true, line_pred in zip(reader_true, reader_pred): + # i, txt, head, label, clabel, nuc, rank + assert line_true[0] == line_pred[0] # safety check + ok_a = line_true[2] == line_pred[2] + ok_l = line_true[4] == line_pred[4] # use clabel + ok_n = line_true[5] == line_pred[5] + ok_r = line_true[6] == line_pred[6] + # update running counters + cnt_tot += 1 + if ok_a: + cnt_a += 1 + if ok_l: + cnt_l += 1 + if ok_n: + cnt_n += 1 + if ok_r: + cnt_r += 1 + if ok_a and ok_l: + cnt_al += 1 + if ok_a and ok_n: + cnt_an += 1 + if ok_a and ok_r: + cnt_ar += 1 + if ok_a and ok_l and ok_n: + cnt_aln += 1 + if ok_a and ok_l and ok_r: + cnt_alr += 1 + if ok_a and ok_l and ok_n and ok_r: + cnt_alnr += 1 + print('\t'.join( + ['{parser_name: <{width}}'.format( + parser_name=author_pred, width=len_author_str)] + + ['{:.4f}'.format(float(cnt_x) / cnt_tot) + for cnt_x in [cnt_a, cnt_l, cnt_n, cnt_r, + cnt_al, cnt_an, cnt_ar, + cnt_aln, cnt_alr, + cnt_alnr]] + + [str(cnt_tot)] + )) diff --git a/evals/feng.py b/evals/feng.py new file mode 100644 index 0000000..a9c60f0 --- /dev/null +++ b/evals/feng.py @@ -0,0 +1,141 @@ +"""Load the output of the RST parser from (Feng and Hirst, 2014). + +This is 99% a copy/paste from evals/joty.py . +I need to come up with a better API and refactor accordingly. +""" + +from __future__ import absolute_import, print_function + +import codecs +import glob +import itertools +import os + +from nltk import Tree + +from educe.rst_dt.deptree import RstDepTree +from educe.rst_dt.parse import parse_rst_dt_tree + + +def load_feng_output_files(root_dir): + """Load ctrees output by Feng & Hirst's parser on the TEST section of + RST-WSJ. + + Parameters + ---------- + root_dir: string + Path to the main folder containing the parser's output + + Returns + ------- + data: dict + Dictionary that should be akin to a sklearn Bunch, with + interesting keys 'filenames', 'doc_names' and 'rst_ctrees'. + + Notes + ----- + To ensure compatibility with the rest of the code base, doc_names + are automatically added the ".out" extension. This would not work + for fileX documents, but they are absent from the TEST section of + the RST-WSJ treebank. + """ + # find all files with the right extension + file_ext = '.txt.dis' + pathname = os.path.join(root_dir, '*{}'.format(file_ext)) + # filenames are sorted by name to avoid having to realign data + # loaded with different functions + filenames = sorted(glob.glob(pathname)) # glob.glob() returns a list + + # find corresponding doc names + doc_names = [os.path.basename(filename).rsplit('.', 2)[0] + '.out' + for filename in filenames] + + # load the RST trees + rst_ctrees = [] + for filename in filenames: + with codecs.open(filename, 'r', 'utf-8') as f: + # TODO (?) add support for and use RSTContext + rst_ctree = parse_rst_dt_tree(f.read(), None) + rst_ctrees.append(rst_ctree) + + data = dict(filenames=filenames, + doc_names=doc_names, + rst_ctrees=rst_ctrees) + + return data + + +def load_feng_ctrees(out_dir, rel_conv): + """Load the ctrees output by Feng's parser as .dis files. + + This currently runs on the document-level files (.doc_dis). + + Parameters + ---------- + out_dir: str + Path to the base directory containing the output files. + + Returns + ------- + ctree_pred: dict(str, RSTTree) + RST ctree for each document. + """ + # load predicted trees + data_pred = load_feng_output_files(out_dir) + # filenames = data_pred['filenames'] + doc_names_pred = data_pred['doc_names'] + rst_ctrees_pred = data_pred['rst_ctrees'] + + # build a dict from doc_name to ctree (RSTTree) + ctree_pred = dict() # constituency trees + for doc_name, ct_pred in itertools.izip(doc_names_pred, rst_ctrees_pred): + # constituency tree + # replace fine-grained labels with coarse-grained labels ; + # the files we have already contain the coarse labels, except their + # initial letter is capitalized whereas ours are not + if rel_conv is not None: + ct_pred = rel_conv(ct_pred) + # "normalize" names of classes of RST relations: + # "textual-organization" => "textual" + for pos in ct_pred.treepositions(): + t = ct_pred[pos] + if isinstance(t, Tree): + node = t.label() + if node.rel == 'textual-organization': + node.rel = 'textual' + # end normalize + ctree_pred[doc_name] = ct_pred + + return ctree_pred + + +def load_feng_dtrees(out_dir, rel_conv, nary_enc='chain'): + """Get the dtrees that correspond to the ctrees output by Feng's parser. + + Parameters + ---------- + out_dir: str + Path to the base directory containing the output files. + nary_enc: one of {'chain', 'tree'} + Encoding for n-ary nodes. + + Returns + ------- + dtree_pred: dict(str, RstDepTree) + RST dtree for each document. + """ + # load predicted c-trees + ctree_pred = load_feng_ctrees(out_dir, rel_conv) + + # build a dict from doc_name to ordered dtree (RstDepTree) + dtree_pred = dict() + for doc_name, ct_pred in ctree_pred.items(): + # convert to an ordered dependency tree ; + # * 'tree' produces a weakly-ordered dtree strictly equivalent + # to the original ctree, + # * 'chain' produces a strictly-ordered dtree for which strict + # equivalence is not preserved + dt_pred = RstDepTree.from_rst_tree(ct_pred, nary_enc=nary_enc) + dtree_pred[doc_name] = dt_pred + + return dtree_pred diff --git a/evals/gcrf_tree_format.py b/evals/gcrf_tree_format.py new file mode 100644 index 0000000..ba8fe27 --- /dev/null +++ b/evals/gcrf_tree_format.py @@ -0,0 +1,222 @@ +"""Module to load .tree files, output by Feng's gCRF parser. + +The .tree files contain binary constituency trees as bracketed strings. +They differ from the .dis files in that the relation label and +nuclearity are written on the top node instead of the daughter nodes, +plus edu spans are not explicitly written at each node. +""" + +from __future__ import absolute_import, print_function +import codecs +from glob import glob +import os +import re + +from nltk.tree import Tree + +from educe.rst_dt.annotation import EDU, Node, SimpleRSTTree, Span +from educe.rst_dt.deptree import RstDepTree + + +TXT_RE = r"(?P.+)_!(?P.+)!_(?P.+)" +TXT_PATTERN = re.compile(TXT_RE, flags=re.DOTALL) + + +def reduce_preterminal(terminals, txt_offset, edu_offset): + """Create a pre-terminal from a list of terminals. + + Parameters + ---------- + terminals: list of str + List of terminals + + Returns + ------- + sct: SimpleRSTTree + Pre-terminal. + """ + edu_num = edu_offset + edu_txt = ' '.join(terminals) + assert edu_txt.startswith('_!') and edu_txt.endswith('!_') + edu_txt = edu_txt[2:-2] # shave off _! and !_ + edu_txt_span = Span(txt_offset, + txt_offset + len(edu_txt)) + edu = EDU(edu_num, edu_txt_span, edu_txt, + context=None, + origin=None) + # "pre-terminal" + pre_node = Node('leaf', (edu_num, edu_num), edu_txt_span, + 'leaf', context=None) + sct = SimpleRSTTree(pre_node, [edu]) + return sct + + +def nltk_to_simple(node, txt_offset=0, edu_offset=1): + """Convert an NLTK Tree to a SimpleRSTTree. + + Parameters + ---------- + node: Tree + Current tree node. + txt_offset: int, defaults to 0 + Current text offset. + edu_offset: int, defaults to 1 + Current EDU id offset. + + Returns + ------- + sct: SimpleRSTTree + Corresponding SimpleRSTTree. + """ + cur_txt_offset = txt_offset + cur_edu_offset = edu_offset + + # first, recurse: convert kids + new_kids = [] + for kid in node: + if isinstance(kid, Tree): + # convert gCRF .tree subtree to SimpleRSTTree + new_kid = nltk_to_simple(kid, txt_offset=cur_txt_offset, + edu_offset=cur_edu_offset) + # update current offsets + cur_txt_offset = new_kid.label().span.char_end + 1 + cur_edu_offset = new_kid.label().edu_span[1] + 1 + new_kids.append(new_kid) + else: + # kid is a terminal + # first, restore parentheses in the text + kid = kid.replace('-LRB-', '(').replace('-RRB-', ')') + # + if not new_kids or isinstance(new_kids[-1], SimpleRSTTree): + new_kids.append([]) + new_kids[-1].append(kid) + if kid.endswith('!_'): + new_kid = reduce_preterminal( + new_kids[-1], cur_txt_offset, cur_edu_offset) + new_kids[-1] = new_kid + # update current offsets + # * txt_offset: + 1 for whitespace or newline + cur_txt_offset = new_kid.label().span.char_end + 1 + # * edu_offset: + 1 for next EDU + cur_edu_offset = new_kid.label().edu_span[1] + 1 + # check that all have been converted + assert all(isinstance(x, SimpleRSTTree) for x in new_kids) + + # we can now compute the label ; the edu_span depends on the + # recursive calls + lbl = node.label() + rel, nuc = lbl.split('[', 1) # nuc = "N][S]" + nuc = nuc[0] + nuc[3] + edu_span = (new_kids[0].label().edu_span[0], + new_kids[-1].label().edu_span[1]) + txt_span = Span(new_kids[0].label().span.char_start, + new_kids[-1].label().span.char_end) + new_lbl = Node(nuc, edu_span, txt_span, rel) + return SimpleRSTTree(new_lbl, new_kids) + + +def _load_gcrf_tree_file(f): + """Do load""" + # replace parentheses in text to avoid confusion with parentheses + # denoting the bracketed tree structure + lines = [] + for line in f: + # replace non-breaking spaces... damn python 2 + if u"\u00a0" in line: + line = line.replace(u"\u00a0", u" ") + # + m = TXT_PATTERN.match(line) + if m is not None: + new_line = (m.group('prefix') + + '_!' + + (m.group('text') + .replace('(', '-LRB-') + .replace(')', '-RRB-')) + + '!_' + + m.group('suffix')) + line = new_line + lines.append(line) + ct_str = ''.join(lines) + ct = Tree.fromstring(ct_str) + sct = nltk_to_simple(ct) + return sct + + +def load_gcrf_tree_file(fname): + """Load a gCRF tree file. + + Parameters + ---------- + fname: str + Path to the file to be loaded. + + Returns + ------- + ct: SimpleRSTTree + Binary constituency tree with relation label and nuclearity + moved one up. + """ + with codecs.open(fname, encoding='utf-8') as f: + ct = _load_gcrf_tree_file(f) + return ct + + +def load_gcrf_ctrees(out_dir, rel_conv): + """Load the ctrees output by gCRF as .tree files. + + Parameters + ---------- + out_dir: str + Path to the base directory containing the output files. + + Returns + ------- + ctree_pred: dict(str, RSTTree) + RST ctree for each document. + """ + ctree_pred = dict() + for f_tree in glob(os.path.join(out_dir, '*.tree')): + doc_name = os.path.splitext(os.path.basename(f_tree))[0] + sct_pred = load_gcrf_tree_file(f_tree) + ct_pred = SimpleRSTTree.to_binary_rst_tree(sct_pred) + if rel_conv is not None: + ct_pred = rel_conv(ct_pred) + # "normalize" names of classes of RST relations: + # "textual-organization" => "textual" + for pos in ct_pred.treepositions(): + t = ct_pred[pos] + if isinstance(t, Tree): + node = t.label() + if node.rel == 'textual-organization': + node.rel = 'textual' + # end normalize + ctree_pred[doc_name] = ct_pred + + return ctree_pred + + +def load_gcrf_dtrees(out_dir, rel_conv, nary_enc='chain', ctree_pred=None): + """Get the dtrees that correspond to the ctrees output by gCRF. + + Parameters + ---------- + out_dir: str + Path to the base directory containing the output files. + nary_enc: one of {'chain', 'tree'} + Encoding for n-ary nodes. + ctree_pred : dict(str, RSTTree), optional + RST c-trees, indexed by doc_name. If c-trees are provided this + way, `out_dir` is ignored. + + Returns + ------- + dtree_pred: dict(str, RstDepTree) + RST dtree for each document. + """ + if ctree_pred is None: + ctree_pred = load_gcrf_ctrees(out_dir, rel_conv) + dtree_pred = dict() + for doc_name, ct_pred in ctree_pred.items(): + dt_pred = RstDepTree.from_rst_tree(ct_pred, nary_enc=nary_enc) + dtree_pred[doc_name] = dt_pred + return dtree_pred diff --git a/evals/hayashi_cons.py b/evals/hayashi_cons.py new file mode 100644 index 0000000..7bdb9a7 --- /dev/null +++ b/evals/hayashi_cons.py @@ -0,0 +1,159 @@ +"""Load RST c-trees output by Hayashi et al.'s reimplementation of HILDA. + +""" + +from __future__ import absolute_import, print_function + +from collections import namedtuple +import codecs +import glob +import itertools +import os + +from nltk import Tree + +from educe.annotation import Span +from educe.rst_dt.annotation import EDU, Node, RSTTree +from educe.rst_dt.deptree import RstDepTree + + +node_struct = namedtuple('node_struct', ['nuc', 'rel', 'span']) + +def read_node(s): + """Helper applied when reading a node""" + nuc, rel = s.split(':') if s != 'Root' else (s, '---') + res = node_struct(nuc=nuc, rel=rel, span=(0, 0)) + return res + + +leaf_struct = namedtuple('leaf_struct', ['edu_id', 'sent_id', 'para_id']) + +def read_leaf(s): + """Helper applied when reading a leaf""" + edu_id, sent_id, para_id = s[4:].split('_') # ex: leaf1_1_1 + res = leaf_struct(edu_id=edu_id, sent_id=sent_id, + para_id=para_id) + return res + +def propagate_spans(t): + """Propagate spans bottom-up in our custom NLTK tree.""" + dft_span = Span(0, 0) # default text span + dft_text = '' + + lbl = t.label() + if all(isinstance(kid, Tree) for kid in t): + new_kids = [propagate_spans(kid) for kid in t] + edu_start = new_kids[0].label().edu_span[0] + edu_end = new_kids[-1].label().edu_span[1] + else: + # pre-terminal + assert len(t) == 1 + kid = t[0] + new_kid = EDU(int(kid.edu_id), dft_span, dft_text) + new_kids = [new_kid] + edu_start = new_kid.num + edu_end = new_kid.num + new_lbl = Node(lbl.nuc, (edu_start, edu_end), dft_span, lbl.rel) + new_tree = RSTTree(new_lbl, new_kids) + return new_tree + + +def load_hayashi_con_files(root_dir): + """Load the ctrees output by Hayashi et al.'s reimplementation of HILDA. + + The RST ctrees are supposedly document-level RST trees, with classes of + relations. + + Parameters + ---------- + out_dir: str + Path to the base directory containing the output files. + + Returns + ------- + data: dict + Dictionary that should be akin to a sklearn Bunch, with + interesting keys 'filenames', 'doc_names' and 'rst_ctrees'. + """ + # map output filename to doc filename + # ex of filename: wsj_0602.out.dis + out_filenames = sorted(glob.glob(os.path.join(root_dir, '*.dis'))) + doc_names = [os.path.basename(out_fn).rsplit('.', 1)[0] + for out_fn in out_filenames] + # load the RST trees + rst_ctrees = [] + for out_fn in out_filenames: + with codecs.open(out_fn, 'r', 'utf-8') as f: + tree_str = f.read() + tree_raw = Tree.fromstring(tree_str, read_node=read_node, + read_leaf=read_leaf) + # TODO(?) add support for and use RSTContext + rst_ctree = propagate_spans(tree_raw) + rst_ctrees.append(rst_ctree) + + data = dict(filenames=out_filenames, + doc_names=doc_names, + rst_ctrees=rst_ctrees) + return data + + +def load_hayashi_hilda_ctrees(out_dir, rel_conv): + """Load the ctrees output by Hayashi et al.'s HILDA. + + Parameters + ---------- + out_dir: str + Path to the folder containing .dis files. + rel_conv: RstRelationConverter + Converter for relation labels (fine- to coarse-grained, plus + normalization). + + Returns + ------- + ctree_pred: dict(str, RSTTree) + RST ctree for each document. + """ + # load predicted ctrees + data_pred = load_hayashi_con_files(out_dir) + doc_names_pred = data_pred['doc_names'] + rst_ctrees_pred = data_pred['rst_ctrees'] + + # build a dict from doc_name to RST ctree + ctree_pred = dict() + for doc_name, ct_pred in itertools.izip(doc_names_pred, rst_ctrees_pred): + if rel_conv is not None: + ct_pred = rel_conv(ct_pred) + ctree_pred[doc_name] = ct_pred + return ctree_pred + + +def load_hayashi_hilda_dtrees(out_dir, rel_conv, nary_enc='chain', + ctree_pred=None): + """Load the dtrees for the ctrees output by Hayashi et al.'s HILDA. + + Parameters + ---------- + out_dir: str + Path to the folder containing .dis files. + rel_conv: RstRelationConverter + Converter for relation labels (fine- to coarse-grained, plus + normalization). + ctree_pred : dict(str, RSTTree), optional + RST c-trees, indexed by doc_name. If c-trees are provided this + way, `out_dir` is ignored. + + Returns + ------- + dtree_pred: dict(str, RstDepTree) + RST dtree for each document. + """ + if ctree_pred is None: + # load predicted ctrees + ctree_pred = load_hayashi_hilda_ctrees(out_dir, rel_conv) + # convert to dtrees + dtree_pred = dict() + for doc_name, ct_pred in ctree_pred.items(): + dt_pred = RstDepTree.from_rst_tree(ct_pred, nary_enc=nary_enc) + dtree_pred[doc_name] = dt_pred + + return dtree_pred diff --git a/evals/hayashi_deps.py b/evals/hayashi_deps.py new file mode 100644 index 0000000..b6f40d2 --- /dev/null +++ b/evals/hayashi_deps.py @@ -0,0 +1,173 @@ +"""Load dependencies output by Hayashi et al.'s parsers. + +This module enables to process files in auto_parse/{dep/li,cons/trans_li}. +""" + +from __future__ import absolute_import, print_function + +import os +from glob import glob + +from educe.learning.edu_input_format import load_edu_input_file +from educe.rst_dt.deptree import RstDepTree, RstDtException +from educe.rst_dt.dep2con import deptree_to_rst_tree + + +def _load_hayashi_dep_file(f, edus): + """Do load. + + Parameters + ---------- + f: File + dep file, open + edus: list of EDU + True EDUs in this document. + + Returns + ------- + dt: RstDepTree + Predicted dtree + """ + dt = RstDepTree(edus=edus, origin=None, nary_enc='chain') # FIXME origin + for line in f: + line = line.strip() + if not line: + continue + dep_idx, gov_idx, lbl = line.split() + dep_idx = int(dep_idx) + gov_idx = int(gov_idx) + dt.add_dependency(gov_idx, dep_idx, label=lbl) + return dt + + +def load_hayashi_dep_file(fname, edus): + """Load a file. + + Parameters + ---------- + fname: str + Path to the file + + Returns + ------- + dt: RstDepTree + Dependency tree corresponding to the content of this file. + """ + with open(fname) as f: + return _load_hayashi_dep_file(f, edus) + + +def load_hayashi_dep_files(out_dir, doc_edus): + """Load dep files output by one of Hayashi et al.'s parser. + + Parameters + ---------- + out_dir: str + Path to the folder containing the .dis files. + doc_edus : dict(str, list(EDU)) + Mapping from doc_name to the list of its EDUs (read from the + corpus). + """ + dtrees = dict() + for fname in glob(os.path.join(out_dir, '*.dis')): + doc_name = os.path.splitext(os.path.basename(fname))[0] + edus = doc_edus[doc_name] + dtrees[doc_name] = load_hayashi_dep_file(fname, edus) + return dtrees + + +def load_hayashi_dep_dtrees(out_dir, rel_conv, doc_edus, edus_file_pat, + nuc_clf, rnk_clf): + """Load the dtrees output by one of Hayashi et al.'s dep parsers. + + Parameters + ---------- + out_dir : str + Path to the folder containing .dis files. + rel_conv : RstRelationConverter + Converter for relation labels (fine- to coarse-grained, plus + normalization). + doc_edus : dict(str, list(EDU)) + Mapping from doc_name to the list of its EDUs (read from the + corpus). + edus_file_pat : str + Pattern for the .edu_input files. + nuc_clf : NuclearityClassifier + Nuclearity classifier + rnk_clf : RankClassifier + Rank classifier + + Returns + ------- + dtree_pred: dict(str, RstDepTree) + RST dtree for each document. + """ + dtree_pred = dict() + + dtrees = load_hayashi_dep_files(out_dir, doc_edus) + for doc_name, dt_pred in dtrees.items(): + if rel_conv is not None: + dt_pred = rel_conv(dt_pred) + # normalize names of classes of RST relations: + # "root" is "ROOT" in my coarse labelset (TODO: make it consistent) + dt_pred.labels = ['ROOT' if x == 'root' else x + for x in dt_pred.labels] + # end normalize + # WIP add nuclearity and rank + edus_data = load_edu_input_file(edus_file_pat.format(doc_name), + edu_type='rst-dt') + edu2sent = edus_data['edu2sent'] + dt_pred.sent_idx = [0] + edu2sent # 0 for fake root ; DIRTY + dt_pred.nucs = nuc_clf.predict([dt_pred])[0] + dt_pred.ranks = rnk_clf.predict([dt_pred])[0] + # end WIP + dtree_pred[doc_name] = dt_pred + + return dtree_pred + + +def load_hayashi_dep_ctrees(out_dir, rel_conv, doc_edus, edus_file_pat, + nuc_clf, rnk_clf, dtree_pred=None): + """Load the ctrees for the dtrees output by one of Hayashi et al.'s + dep parsers. + + Parameters + ---------- + out_dir : str + Path to the folder containing .dis files. + rel_conv : RstRelationConverter + Converter for relation labels (fine- to coarse-grained, plus + normalization). + doc_edus : dict(str, list(EDU)) + Mapping from doc_name to the list of its EDUs (read from the + corpus). + edus_file_pat : str + Pattern for the .edu_input files. + nuc_clf : NuclearityClassifier + Nuclearity classifier + rnk_clf : RankClassifier + Rank classifier + dtree_pred : dict(str, RstDepTree), optional + RST d-trees, indexed by doc_name. If d-trees are provided this + way, `out_dir` is ignored. + + Returns + ------- + ctree_pred: dict(str, RSTTree) + RST ctree for each document. + """ + ctree_pred = dict() + if dtree_pred is None: + dtree_pred = load_hayashi_dep_dtrees(out_dir, rel_conv, doc_edus, + edus_file_pat, + nuc_clf, rnk_clf) + for doc_name, dt_pred in dtree_pred.items(): + try: + ct_pred = deptree_to_rst_tree(dt_pred) + except RstDtException: + print(doc_name) + raise + else: + ctree_pred[doc_name] = ct_pred + + return ctree_pred diff --git a/evals/ji.py b/evals/ji.py new file mode 100644 index 0000000..1b1808c --- /dev/null +++ b/evals/ji.py @@ -0,0 +1,181 @@ +"""Load the output of Ji's DPLP parser. + +""" + +from __future__ import absolute_import, print_function + +from collections import defaultdict +from glob import glob +import os + +from educe.annotation import Span +from educe.corpus import FileId +from educe.rst_dt.annotation import Node, RSTTree +from educe.rst_dt.deptree import RstDepTree + + +def load_ji_ctrees(ji_out_dir, rel_conv, doc_edus): + """Load the ctrees output by DPLP as .brackets files. + + Parameters + ---------- + ji_out_dir : str + Path to the base directory containing the output files. + rel_conv : RstRelationConverter? + Relation converter. + doc_edus : dict(str, list(EDU)) + Mapping from doc_name to the list of its EDUs (read from the + corpus). + + Returns + ------- + ctree_pred: dict(str, RSTTree) + RST ctree for each document. + """ + # FIXME? get the text of EDUs from the .merge files? + # * for each doc, load the predicted spans from the .brackets + ctree_pred = dict() + files_pred = os.path.join(ji_out_dir, '*.brackets') + for f_pred in sorted(glob(files_pred)): + doc_name = os.path.splitext(os.path.basename(f_pred))[0] + edus = {i: e for i, e in enumerate(doc_edus[doc_name], start=1)} + origin = FileId(doc_name, None, None, None) + # read spans + spans_pred = defaultdict(list) # predicted spans by length + with open(f_pred) as f: + for line in f: + # FIXME use a standard module: ast? pickle? + # * drop surrounding brackets + opening bracket of edu span + line = line.strip()[2:-1] + edu_span, nuc_rel = line.split('), ') + edu_span = tuple(int(x) for x in edu_span.split(', ')) + nuc, rel = nuc_rel.split(', ') + # * remove quotes around nuc and rel + nuc = nuc[1:-1] + rel = rel[1:-1] + # + edu_span_len = edu_span[1] - edu_span[0] + spans_pred[edu_span_len].append((edu_span, nuc, rel)) + # bottom-up construction of the RST ctree + # left_border -> list of RST ctree fragments, sorted by len + tree_frags = defaultdict(list) + for span_len, spans in sorted(spans_pred.items()): + for edu_span, nuc, rel in spans: + children = [] + edu_beg, edu_end = edu_span + if edu_beg == edu_end: + # pre-terminal + txt_span = edus[edu_beg].span + # one child: leaf node: EDU + leaf = edus[edu_beg] + children.append(leaf) + else: + # internal node + # * get the children (subtrees) + edu_cur = edu_beg + while edu_cur <= edu_end: + kid_nxt = tree_frags[edu_cur][-1] + children.append(kid_nxt) + edu_cur = kid_nxt.label().edu_span[1] + 1 + # compute properties of this node + txt_span = Span(children[0].label().span.char_start, + children[-1].label().span.char_end) + # build node and RSTTree fragment + node = Node(nuc, edu_span, txt_span, rel, + context=None) # TODO context? + tree_frags[edu_beg].append( + RSTTree(node, children, origin=origin)) + # build the top node + edu_nums = sorted(edus.keys()) + edu_span = (edu_nums[0], edu_nums[-1]) + children = [] + edu_beg, edu_end = edu_span + edu_cur = edu_beg + while edu_cur <= edu_end: + kid_nxt = tree_frags[edu_cur][-1] + children.append(kid_nxt) + edu_cur = kid_nxt.label().edu_span[1] + 1 + txt_span = Span(children[0].label().span.char_start, + children[-1].label().span.char_end) + node = Node(nuc, edu_span, txt_span, 'Root', context=None) + tree_frags[edu_beg].append( + RSTTree(node, children, origin=origin)) + # now we should have a spanning ctree + ct_pred = tree_frags[1][-1] + assert ct_pred.label().edu_span == (sorted(edus.keys())[0], + sorted(edus.keys())[-1]) + # convert relation labels + if rel_conv is not None: + ct_pred = rel_conv(ct_pred) + # normalize names of classes of RST relations: + # "same_unit" => "same-unit" + # "topic" => "topic-change" or "topic-comment"? + for pos in ct_pred.treepositions(): + t = ct_pred[pos] + if isinstance(t, RSTTree): + node = t.label() + # replace "same_unit" with "same-unit" + if node.rel == 'same_unit': # DPLP v. 1 + node.rel = 'same-unit' + elif node.rel == 'topic': # DPLP v. 1 + # either "topic-comment" or "topic-change" ; + # I expect the parser to find "topic-comment" to + # be easier but apparently it has no consequence + # on the current output I reproduced + node.rel = 'topic-comment' + elif node.rel == 'sameunit': # Ji's output + node.rel = 'same-unit' + elif node.rel == 'topicchange': # Ji's output + node.rel = 'topic-change' + elif node.rel == 'topiccomment': # Ji's output + node.rel = 'topic-comment' + elif node.rel == 'textual-organization': # WLW17 output + # we use 'textual' as the coarse label ; + # JE14 outputs textualorganization which is the + # fine label in our taxonomy, hence is mapped to + # textual beforehand + node.rel = 'textual' + # end normalize + # store the resulting RSTTree + ctree_pred[doc_name] = ct_pred + + return ctree_pred + + +def load_ji_dtrees(ji_out_dir, rel_conv, doc_edus, nary_enc='chain', + ctree_pred=None): + """Get the dtrees that correspond to the ctrees output by DPLP. + + Parameters + ---------- + ji_out_dir: str + Path to the base directory containing the output files. + rel_conv: TODO + Relation converter, from fine- to coarse-grained labels. + nary_enc: one of {'chain', 'tree'} + Encoding for n-ary nodes. + doc_edus : dict(str, list(EDU)) + Mapping from doc_name to the list of its EDUs (read from the + corpus). + ctree_pred : dict(str, RSTTree), optional + RST c-trees, indexed by doc_name. If c-trees are provided this + way, `out_dir` is ignored. + + Returns + ------- + dtree_pred: dict(str, RstDepTree) + RST dtree for each document. + """ + dtree_pred = dict() + if ctree_pred is None: + ctree_pred = load_ji_ctrees(ji_out_dir, rel_conv, doc_edus) + for doc_name, ct_pred in ctree_pred.items(): + dtree_pred[doc_name] = RstDepTree.from_rst_tree( + ct_pred, nary_enc=nary_enc) + # set reference to the document in the RstDepTree (required by + # dump_disdep_files) + for doc_name, dt_pred in dtree_pred.items(): + dt_pred.origin = FileId(doc_name, None, None, None) + + return dtree_pred + diff --git a/evals/li2014.py b/evals/li2014.py new file mode 100644 index 0000000..1135efc --- /dev/null +++ b/evals/li2014.py @@ -0,0 +1,122 @@ +"""Evaluation procedure used in the parser of (Li et al. 2014). + +This is a reimplementation of this evaluation procedure. +""" + +from educe.rst_dt.metrics.rst_parseval import (rst_parseval_report, + rst_parseval_detailed_report) + + + +# FIXME legacy code brutally dumped here, broken +def twisted_eval_li2014(data_true, data_pred): + """Run Parseval on transformed gold trees, as in (Li et al., 2014). + + This applies a deterministic transform to the gold constituency tree + that basically re-orders attachments of a head EDU. + """ + # 1. ctrees_true -> dtrees_true or dtrees_twis (if the procedure + # is fishy) + # 2. dtrees_[true|twis] -> ctrees_twis + # RESUME HERE + # hint: ctrees_twis contain only NS nuclearity (...) + + # TODO check exact conformance with the code of their parser: + # how rank and nuclearity are determined + data_true['rst_ctrees'] = [] + for dt_true in data_true['rst_dtrees']: + # FIXME map EDUs to sentences + dt_true.sent_idx = [edu_id2sent_idx[e.identifier()] + for e in dt_true.edus] + # TODO check that 'lllrrr' effectively corresponds to the strategy + # they apply + chn_bin_srtree_true = deptree_to_simple_rst_tree( + dt_true, MULTINUC_LBLS, strategy='lllrrr') + chn_bin_rtree_true = SimpleRSTTree.to_binary_rst_tree( + chn_bin_srtree_true) + bin_rtree_true = chn_bin_rtree_true + data_true['rst_ctrees'].append(bin_rtree_true) +# end FIXME + + +# FIXME currently broken, need to declare and fit classifiers for nuc and rank +# (nuc_classifier and rank_classifier) +# TODO move to ? +def eval_distortion_gold(corpus, nuc_strategy, rank_strategy, + prioritize_same_unit): + """Load an RstDepTree from the output of attelo. + + Parameters + ---------- + corpus: string + Path to the gold corpus to be evaluated + nuc_strategy: string + Strategy to predict nuclearity + rank_strategy: string + Strategy to predict attachment ranking + """ + # print parameters + print('corpus: {}\tnuc_strategy: {}\trank_strategy: {}'.format( + corpus, nuc_strategy, rank_strategy)) + + gold_orig = dict() + gold_twis = dict() + + # FIXME: find ways to read the right (not necessarily TEST) section + # and only the required documents + rst_reader = RstReader(corpus) + rst_corpus = rst_reader.slurp() + for doc_id, rtree_ref in sorted(rst_corpus.items()): + doc_name = doc_id.doc + + # original gold + # convert labels to coarse + coarse_rtree_ref = REL_CONV(rtree_ref) + # convert to binary tree + bin_rtree_ref = _binarize(coarse_rtree_ref) + gold_orig[doc_name] = bin_rtree_ref + + # distorted gold: forget nuclearity and order of attachment + # convert to RstDepTree via SimpleRSTTree + bin_srtree_ref = SimpleRSTTree.from_rst_tree(coarse_rtree_ref) + dt_ref = RstDepTree.from_simple_rst_tree(bin_srtree_ref) + # FIXME replace gold nuclearity and rank with predicted ones, + # using the given heuristics + # dt_ref.nucs = nuc_classifier.predict([dt_ref])[0] + # dt_ref.ranks = rank_classifier.predict([dt_ref])[0] + # end FIXME + # regenerate a binary RST tree + chn_bin_srtree_ref = deptree_to_simple_rst_tree(dt_ref) + chn_bin_rtree_ref = SimpleRSTTree.to_binary_rst_tree( + chn_bin_srtree_ref) + gold_twis[doc_name] = chn_bin_rtree_ref + + print(rst_parseval_report(gold_orig, gold_twis, + metric_types=[x[0] for x in LBL_FNS], + digits=4)) + # detailed report on S+N+R + print(rst_parseval_detailed_report(ctree_true, ctree_pred, + metric_type='S+R')) + + +def comparative_distortion_on_gold(): + """Evaluate the impact of forgetting nuclearity and rank in the gold. + + Quantify the distortion and loss when forgetting nuclearity and rank + in the gold and replacing them with deterministically-determined + values. + + Possible configurations are the cross-product of strategies to + heuristically determine rank and nuclearity. + """ + gold_corpus = CD_TRAIN # CD_TEST + nuc_strats = ["most_frequent_by_rel", + "unamb_else_most_frequent"] + rank_strats = ['lllrrr', + 'rrrlll', + 'lrlrlr', + 'rlrlrl'] + prioritize_same_units = [True, False] + for nuc_strat in nuc_strats: + for rank_strat in rank_strats: + eval_distortion_gold(gold_corpus, nuc_strat, rank_strat) diff --git a/evals/li_qi.py b/evals/li_qi.py new file mode 100644 index 0000000..2df67d2 --- /dev/null +++ b/evals/li_qi.py @@ -0,0 +1,137 @@ +"""Load the output of the parser from (Li et al. 2016). + +This is 99% a copy/paste from our own evals/joty.py. +I really, really need to come up with a better API and refactor accordingly. +""" + +from __future__ import absolute_import, print_function + +import codecs +import glob +import itertools +import os + +from educe.rst_dt.parse import parse_rst_dt_tree +from educe.rst_dt.deptree import RstDepTree + + +def load_li_qi_output_files(root_dir): + """Load ctrees output by Li Qi's parser on the TEST section of the RST-DT. + + Parameters + ---------- + root_dir: string + Path to the main folder containing the parser's output + + Returns + ------- + data: dict + Dictionary that should be akin to a sklearn Bunch, with + interesting keys 'filenames', 'doc_names' and 'rst_ctrees'. + + Notes + ----- + To ensure compatibility with the rest of the code base, doc_names + are automatically added the ".out" extension. This would not work + for fileX documents, but they are absent from the TEST section of + the RST-WSJ treebank. + """ + # map output filename to doc filename: + # here, remove prefix "parsed_" + # ex of filename: parsed_wsj_0602.out + out_filenames = sorted(glob.glob(os.path.join(root_dir, 'parsed_*'))) + doc_names = [os.path.basename(out_fn).split('_', 1)[1] + for out_fn in out_filenames] + # load the RST trees + rst_ctrees = [] + for out_fn in out_filenames: + with codecs.open(out_fn, 'r', 'utf-8') as f: + # TODO(?) add support for and use RSTContext + rst_ctree = parse_rst_dt_tree(f.read(), None) + rst_ctrees.append(rst_ctree) + + data = dict(filenames=out_filenames, + doc_names=doc_names, + rst_ctrees=rst_ctrees) + return data + + +def load_li_qi_ctrees(out_dir, rel_conv): + """Load the ctrees output by Li Qi's parser as .dis files. + + This currently runs on the document-level files (.doc_dis). + + Parameters + ---------- + out_dir: str + Path to the base directory containing the output files. + + Returns + ------- + ctree_pred: dict(str, RSTTree) + RST ctree for each document. + """ + # load predicted trees + data_pred = load_li_qi_output_files(out_dir) + doc_names_pred = data_pred['doc_names'] + rst_ctrees_pred = data_pred['rst_ctrees'] + # map doc_name to ctree (RSTTree) + ctree_pred = dict() + for doc_name, ct_pred in itertools.izip(doc_names_pred, rst_ctrees_pred): + # ctree + # replace fine-grained labels with coarse-grained labels : + # the files we have already contain the coarse labels, except their + # initial letter is capitalized, except for same-unit and span, + # whereas ours are not + if rel_conv is not None: + ct_pred = rel_conv(ct_pred) + ctree_pred[doc_name] = ct_pred + + return ctree_pred + + +def load_li_qi_dtrees(out_dir, rel_conv, nary_enc='chain', ctree_pred=None): + """Get the dtrees that correspond to the ctrees output by Li Qi's parser. + + Parameters + ---------- + out_dir: str + Path to the base directory containing the output files. + nary_enc: one of {'chain', 'tree'} + Encoding for n-ary nodes. + ctree_pred : dict(str, RSTTree), optional + RST c-trees, indexed by doc_name. If c-trees are provided this + way, `out_dir` is ignored. + + Returns + ------- + dtree_pred: dict(str, RstDepTree) + RST dtree for each document. + """ + if ctree_pred is None: + # load predicted trees + data_pred = load_li_qi_output_files(out_dir) + # filenames = data_pred['filenames'] + doc_names_pred = data_pred['doc_names'] + rst_ctrees_pred = data_pred['rst_ctrees'] + ctree_pred = {doc_name: ct_pred for doc_name, ct_pred + in itertools.izip(doc_names_pred, rst_ctrees_pred)} + # build a dict from doc_name to ordered dtree (RstDepTree) + dtree_pred = dict() + for doc_name, ct_pred in ctree_pred.items(): + # constituency tree + # replace fine-grained labels with coarse-grained labels ; + # the files we have already contain the coarse labels, except their + # initial letter is capitalized whereas ours are not + if rel_conv is not None: + ct_pred = rel_conv(ct_pred) + # convert to an ordered dependency tree ; + # * 'tree' produces a weakly-ordered dtree strictly equivalent + # to the original ctree, + # * 'chain' produces a strictly-ordered dtree for which strict + # equivalence is not preserved + dt_pred = RstDepTree.from_rst_tree(ct_pred, nary_enc=nary_enc) + dtree_pred[doc_name] = dt_pred + + return dtree_pred + diff --git a/evals/li_sujian.py b/evals/li_sujian.py new file mode 100644 index 0000000..d84ae9f --- /dev/null +++ b/evals/li_sujian.py @@ -0,0 +1,278 @@ +"""TODO + +""" + +from __future__ import absolute_import, print_function +import os + +# educe +from educe.learning.edu_input_format import load_edu_input_file +from educe.rst_dt.dep2con import deptree_to_rst_tree +from educe.rst_dt.deptree import NUC_S, RstDepTree, RstDtException +from educe.rst_dt.metrics.rst_parseval import rst_parseval_report +# attelo +from attelo.metrics.deptree import compute_uas_las as att_compute_uas_las + + +# output of Li et al.'s parser +SAVE_DIR = "/home/mmorey/melodi/rst/replication/li_sujian/TextLevelDiscourseParser/mybackup/mstparser-code-116-trunk/mstparser/save" +COARSE_FILES = [ + "136.0detailedOutVersion2.txt", + "151.0detailedOut.txt", + "164.0detailedOut.txt", + "177.0detailedOut.txt", + "335.0detailedOut.txt", + "37.0detailedOut.txt", + "424.0detailedOut.txt", + "448.0detailedOut.txt", + "455.0detailedOutVersion2.txt", + "513.0detailedOutVersion2.txt", + "529.0detailedOut.txt", + "615.0detailedOutVersion2.txt", + "712.0detailedOut.txt", + "917.0detailedOut.txt", +] +FINE_FILES = [ + "190.0detailedOut.txt", + "473.0detailedOutVersion2.txt", + "561.0detailedOut.txt", + "723.0detailedOut.txt", + "747.0detailedOutVersion2.txt", + "825.0detailedOut.txt", + "947.0detailedOut.txt", + "965.0detailedOutVersion2.txt", +] +# different format for predicted labels and description of EDU +COARSE_FEAT_FILES = [ + "441.0detailedOut.txt", +] + +# default file to include ; I picked a coarse-grained one with good scores +DEFAULT_FILE = os.path.join(SAVE_DIR, "712.0detailedOut.txt") + + +def load_output_file(out_file): + """Load an output file from Li et al.'s dep parser. + """ + doc_names = [] + heads_true = [] + labels_true = [] + heads_pred = [] + labels_pred = [] + with open(out_file) as f: + for line in f: + if line.startswith(".\\testdata"): + # file + doc_name = line.strip().split("\\")[2][:12] # drop .edus or else + # print(doc_name) + doc_names.append(doc_name) + heads_true.append([-1]) # initial pad for fake root + labels_true.append(['']) + heads_pred.append([-1]) + labels_pred.append(['']) + else: + edu_idx, hd_true, hd_pred, lbl_true, lbl_pred, edu_str = line.strip().split(' ', 5) + if lbl_pred == '': + # not sure whether this should be enabled + lbl_pred = 'Elaboration' + heads_true[-1].append(int(hd_true)) + labels_true[-1].append(lbl_true) + heads_pred[-1].append(int(hd_pred)) + labels_pred[-1].append(lbl_pred) + res = { + 'doc_names': doc_names, + 'heads_true': heads_true, + 'labels_true': labels_true, + 'heads_pred': heads_pred, + 'labels_pred': labels_pred, + } + return res + + +def load_li_sujian_dep_dtrees(out_file, rel_conv_dtree, edus_file_pat, + nuc_clf, rnk_clf): + """Load the dtrees output by Li Sujian et al.'s dep parser. + + Parameters + ---------- + out_file : str + Path to the file containing all the predictions. + + rel_conv_dtree : RstRelationConverter + Converter to map relation labels to (normalized) coarse-grained + classes. + + edus_file_pat : str + Pattern for the .edu_input files. + + nuc_clf : NuclearityClassifier + Nuclearity classifier + + rnk_clf : RankClassifier + Rank classifier + + Returns + ------- + dtree_pred : dict(str, RstDepTree) + RST dtree for each doc. + """ + dtree_pred = dict() + + dep_bunch = load_output_file(out_file) + # load and process _pred + for doc_name, heads_pred, labels_pred in zip( + dep_bunch['doc_names'], dep_bunch['heads_pred'], + dep_bunch['labels_pred']): + # create dtree _pred + edus_data = load_edu_input_file(edus_file_pat.format(doc_name), + edu_type='rst-dt') + edus = edus_data['edus'] + edu2sent = edus_data['edu2sent'] + dt_pred = RstDepTree(edus) + # add predicted edges + for dep_idx, (gov_idx, lbl) in enumerate(zip( + heads_pred[1:], labels_pred[1:]), start=1): + if lbl == '': + lbl = 'Elaboration' + lbl = lbl.lower() + dt_pred.add_dependency(gov_idx, dep_idx, lbl) + # map to relation classes + dt_pred = rel_conv_dtree(dt_pred) + dt_pred.labels = ['ROOT' if x == 'root' else x + for x in dt_pred.labels] + # attach edu2sent, for later use by rnk_clf + dt_pred.sent_idx = [0] + edu2sent # 0 for fake root + dirty + dtree_pred[doc_name] = dt_pred + # end WIP + + for doc_name in sorted(dtree_pred.keys()): + dt_pred = dtree_pred[doc_name] + # enrich d-tree with nuc and order + dt_pred.ranks = rnk_clf.predict([dt_pred])[0] + dt_pred.nucs = nuc_clf.predict([dt_pred])[0] + dtree_pred[doc_name] = dt_pred + + return dtree_pred + + +def load_li_sujian_dep_ctrees(out_file, rel_conv_dtree, edus_file_pat, + nuc_clf, rnk_clf): + """Load the ctrees for the dtrees output by Li Sujian et al.'s parser. + + Parameters + ---------- + out_file : str + Path to the file containing all the predictions. + + rel_conv_dtree : RstRelationConverter + Converter to map relation labels to (normalized) coarse-grained + classes. + + edus_file_pat : str + Pattern for the .edu_input files. + + nuc_clf : NuclearityClassifier + Nuclearity classifier + + rnk_clf : RankClassifier + Rank classifier + + Returns + ------- + ctree_pred : dict(str, RSTTree) + RST ctree for each doc. + """ + ctree_pred = dict() + + dtree_pred = load_li_sujian_dep_dtrees( + out_file, rel_conv_dtree, edus_file_pat, nuc_clf, rnk_clf) + for doc_name, dt_pred in sorted(dtree_pred.items()): + ct_pred = deptree_to_rst_tree(dt_pred) + ctree_pred[doc_name] = ct_pred + return ctree_pred + + +def twisted_eval(out_file, rel_conv_dtree, setup_dtree_postprocessor, + ctree_true, dtree_true, edus_file_pat): + """Perform a twisted eval. + + Parameters + ---------- + setup_dtree_postprocessor : function + Function that sets up nuc_clf and rnk_clf. + + ctree_true : dict(str, RSTTree) + Gold ctrees + + dtree_true : dict(str, DepRstTree) + Gold dtrees + + out_file : str + Path to the output file. + """ + # setup conversion from c- to d-tree and back, and eval type + nary_enc = 'chain' + # reconstruction of the c-tree + order = 'strict' + nuc_strategy = 'constant' + nuc_constant = NUC_S + rnk_strategy = 'lllrrr' + rnk_prioritize_same_unit = False + # eval + add_trivial_spans = True + + nuc_clf, rnk_clf = setup_dtree_postprocessor( + nary_enc=nary_enc, order=order, nuc_strategy=nuc_strategy, + nuc_constant=nuc_constant, rnk_strategy=rnk_strategy, + rnk_prioritize_same_unit=rnk_prioritize_same_unit) + + ctree_true = dict() + dtree_true = dict() + for doc_name, dt_true in sorted(dtree_true.items()): + # dirty hack: lowercase ROOT + dt_true.labels = [x.lower() if x == 'ROOT' else x + for x in dt_true.labels] + + # load parser output + dtree_pred = load_li_sujian_dep_dtrees( + out_file, rel_conv_dtree, edus_file_pat, nuc_clf, rnk_clf) + ctree_pred = load_li_sujian_dep_ctrees( + out_file, rel_conv_dtree, edus_file_pat, nuc_clf, rnk_clf) + + # use our heuristics to replace the true nuc and order in + # dt_true with a predicted one, replace ct_true with its + # twisted version + for doc_name, dt_true in dtree_true.items(): + dt_pred = dtree_pred[doc_name] + # twiste dt_true + dt_true.sent_idx = dt_pred.sent_idx + dt_true.ranks = rnk_clf.predict([dt_true])[0] + dt_true.nucs = nuc_clf.predict([dt_true])[0] + # re-gen ct_true + try: + ct_true = deptree_to_rst_tree(dt_true) + except RstDtException as rst_e: + print(rst_e) + raise + ctree_true[doc_name] = ct_true + + # compute UAS and LAS on the _true values from the corpus and + # _pred Educe RstDepTrees re-built from their output files + doc_names = sorted(dtree_true.keys()) + dtree_true_list = [dtree_true[doc_name] for doc_name in doc_names] + dtree_pred_list = [dtree_pred[doc_name] for doc_name in doc_names] + sc_uas, sc_las, sc_las_n, sc_las_o, sc_las_no = att_compute_uas_las( + dtree_true_list, dtree_pred_list, include_ls=False, + include_las_n_o_no=True) + print(("{}\tUAS={:.4f}\tLAS={:.4f}\tLAS+N={:.4f}\tLAS+O={:.4f}\t" + "LAS+N+O={:.4f}").format( + out_file, sc_uas, sc_las, sc_las_n, sc_las_o, sc_las_no)) + + # compute RST-Parseval of these c-trees + ctree_true_list = [ctree_true[doc_name] for doc_name in doc_names] + ctree_pred_list = [ctree_pred[doc_name] for doc_name in doc_names] + print(rst_parseval_report(ctree_true_list, ctree_pred_list, + ctree_type='RST', digits=4, + per_doc=False, + add_trivial_spans=add_trivial_spans, + stringent=False)) diff --git a/evals/ours.py b/evals/ours.py new file mode 100644 index 0000000..5a8f210 --- /dev/null +++ b/evals/ours.py @@ -0,0 +1,218 @@ +"""Evaluate our parsers. + +""" + +from __future__ import print_function + +from collections import defaultdict + +import numpy as np + +from educe.annotation import Span as EduceSpan +from educe.rst_dt.annotation import (EDU as EduceEDU, SimpleRSTTree) +from educe.rst_dt.corpus import mk_key +from educe.rst_dt.dep2con import (deptree_to_simple_rst_tree, + deptree_to_rst_tree) +from educe.rst_dt.deptree import RstDepTree, RstDtException +from educe.rst_dt.document_plus import align_edus_with_paragraphs +# +from attelo.io import load_edus +from attelo.table import UNRELATED # for load_attelo_output_file + + +# move to attelo.datasets.attelo_out_format +def load_attelo_output_file(output_file): + """Load edges from an attelo output file. + + An attelo output file typically contains edges from several + documents. This function indexes edges by the name of their + document. + + Parameters + ---------- + output_file: string + Path to the attelo output file + + Returns + ------- + edges_pred: dict(string, [(string, string, string)]) + Predicted edges for each document, indexed by doc name + + Notes + ----- + See `attelo.io.load_predictions` that is almost equivalent to this + function. They are expected to converge some day into a better, + obvious in retrospect, function. + """ + edges_pred = defaultdict(list) + with open(output_file) as f: + for line in f: + src_id, tgt_id, lbl = line.strip().split('\t') + if lbl != UNRELATED: + # dirty hack: get doc name from EDU id + # e.g. (EDU id = wsj_0601_1) => (doc id = wsj_0601) + doc_name = tgt_id.rsplit('_', 1)[0] + edges_pred[doc_name].append((src_id, tgt_id, lbl)) + + return edges_pred + + +def load_attelo_dtrees(output_file, edus_file, rel_clf, nuc_clf, rnk_clf, + doc_edus=None): + """Load RST dtrees from attelo output files. + + Parameters + ---------- + output_file: string + Path to the file that contains attelo's output + edus_file: string + Path to the file that describes EDUs. + doc_edus : dict(str, list(EDU)), optional + Mapping from doc_name to the list of its EDUs (read from the + corpus). If None, each EDU is re-created using information in + the `.edu_input` file, otherwise EDUs are created but their text + is taken from `doc_edus`. + FIXME avoid creating "new" EDUs altogether if `doc_edus` is not + None. + + Returns + ------- + TODO + """ + dtree_pred = dict() # predicted dtrees + # * setup... + # load EDUs as they are known to attelo (sigh): rebuild educe EDUs + # from their attelo description and group them by doc_name + educe_edus = defaultdict(list) + edu2sent_idx = defaultdict(dict) + gid2num = dict() + att_edus = load_edus(edus_file) + for att_edu in att_edus: + # doc name + doc_name = att_edu.grouping + # EDU info + edu_num = int(att_edu.id.rsplit('_', 1)[1]) + edu_span = EduceSpan(att_edu.start, att_edu.end) + if doc_edus is not None: + edu_text = doc_edus[doc_name][edu_num - 1].raw_text + else: + edu_text = att_edu.text + educe_edus[doc_name].append(EduceEDU(edu_num, edu_span, edu_text)) + # map global id of EDU to num of EDU inside doc + gid2num[att_edu.id] = edu_num + # map EDU to sentence + sent_idx = int(att_edu.subgrouping.split('_sent')[1]) + edu2sent_idx[doc_name][edu_num] = sent_idx + # sort EDUs by num + educe_edus = {doc_name: sorted(edus, key=lambda e: e.num) + for doc_name, edus in educe_edus.items()} + # rebuild educe-style edu2sent ; prepend 0 for the fake root + doc_name2edu2sent = {doc_name: ([0] + + [edu2sent_idx[doc_name][e.num] + for e in doc_educe_edus]) + for doc_name, doc_educe_edus in educe_edus.items()} + + # load predicted edges, on these EDUs, into RstDepTrees + edges_pred = load_attelo_output_file(output_file) + for doc_name, es_pred in sorted(edges_pred.items()): + # get educe EDUs + doc_educe_edus = educe_edus[doc_name] + # create pred dtree + dt_pred = RstDepTree(doc_educe_edus) + for src_id, tgt_id, lbl in es_pred: + if src_id == 'ROOT': + if lbl == 'ROOT': + dt_pred.set_root(gid2num[tgt_id]) + else: + raise ValueError('Weird root label: {}'.format(lbl)) + else: + dt_pred.add_dependency(gid2num[src_id], gid2num[tgt_id], lbl) + dt_pred.origin = mk_key(doc_name) + # 2017-12-14 relabel relations + if rel_clf is not None: + dt_pred.labels = rel_clf.predict([dt_pred])[0] + # end relabel relations + # add nuclearity: heuristic baseline WIP or true classifier + dt_pred.nucs = nuc_clf.predict([dt_pred])[0] + # add rank: heuristic baseline, needs edu2sent + edu2sent = doc_name2edu2sent[doc_name] + dt_pred.sent_idx = edu2sent # DIRTY + dt_pred.ranks = rnk_clf.predict([dt_pred])[0] + # store + dtree_pred[doc_name] = dt_pred + + return dtree_pred + + +def load_attelo_ctrees(output_file, edus_file, rel_clf, nuc_clf, rnk_clf, + doc_edus=None, dtree_pred=None): + """Load RST ctrees from attelo output files. + + Parameters + ---------- + output_file: string + Path to the file that contains attelo's output + edus_file: string + Path to the file that describes EDUs. + nuc_clf: NuclearityClassifier + Classifier to predict nuclearity + rnk_clf: RankClassifier + Classifier to predict attachment ranking + doc_edus : dict(str, list(EDU)), optional + Mapping from doc_name to the list of its EDUs (read from the + corpus). If None, each EDU is re-created using information in + the `.edu_input` file, otherwise EDUs are created but their text + is taken from `doc_edus`. + FIXME avoid creating "new" EDUs altogether if `doc_edus` is not + None. + dtree_pred : dict(str, RstDepTree), optional + RST d-trees, indexed by doc_name. If d-trees are provided this + way, `out_dir` is ignored. + + Returns + ------- + TODO + """ + if dtree_pred is None: + # load RST dtrees, with heuristics for nuc and rank + dtree_pred = load_attelo_dtrees(output_file, edus_file, + rel_clf, nuc_clf, rnk_clf, + doc_edus=doc_edus) + # convert to RST ctrees + ctree_pred = dict() + for doc_name, dt_pred in dtree_pred.items(): + try: + rtree_pred = deptree_to_rst_tree(dt_pred) + ctree_pred[doc_name] = rtree_pred + except RstDtException as rst_e: + print(rst_e) + if False: + print('\n'.join('{}: {}'.format(edu.text_span(), edu) + for edu in educe_edus[doc_name])) + # raise + + return ctree_pred + + +def load_deptrees_from_attelo_output(ctree_true, dtree_true, + output_file, edus_file, + nuc_clf, rnk_clf): + """Load an RstDepTree from the output of attelo. + + Parameters + ---------- + ctree_true: dict(str, RSTTree) + Ground truth RST ctree. + dtree_true: dict(str, RstDepTree) + Ground truth RST (ordered) dtree. + skpd_docs: set(string) + Names of documents that should be skipped to compute scores + + Returns + ------- + skipped_docs: set(string) + Names of documents that have been skipped to compute scores + """ + # USE TO INCORPORATE CONSTITUENCY LOSS INTO STRUCTURED CLASSIFIERS + # load predicted trees + # end USE TO INCORPORATE CONSTITUENCY LOSS INTO STRUCTURED CLASSIFIERS diff --git a/evals/prepare_nuc_dataset.py b/evals/prepare_nuc_dataset.py new file mode 100644 index 0000000..e9d534f --- /dev/null +++ b/evals/prepare_nuc_dataset.py @@ -0,0 +1,279 @@ +"""This utility script outputs a dataset of the nuclearity of RST edges. + +Given the path to the RST-DT corpus and a dataset of candidate RST +dependencies labelled with their gold coarse (class) RST relation (or +none if they are unrelated), produce a similar dataset for the task +of nuclearity prediction. + +As of 2017-12-08, we filter out the instances for unrelated pairs of EDUs +and left-oriented dependencies, only keeping right-oriented dependencies +(except for "ROOT"). +The resulting dataset describes a binary classification problem. +""" + +from __future__ import absolute_import, print_function + +import argparse +import codecs +import itertools +import os + +from educe.rst_dt.annotation import NUC_N, NUC_S +from educe.rst_dt.corpus import RstRelationConverter, RELMAP_112_18_FILE +from educe.rst_dt.dep_corpus import read_corpus +from educe.rst_dt.deptree import RstDepTree + + +def main(corpus, dataset, out_dir, nary_enc, model_split): + """Do prepare the nuclearity dataset. + + Parameters + ---------- + corpus : str + Path to the RST-DT "main" corpus. + dataset : str + Path to the existing dataset labelled with coarse relations. + out_dir : str + Path to the output folder. + nary_enc : str, one of {'chain', 'tree'} + Encoding for n-ary nodes. + model_split : str, one of {'none', 'sent', 'sent-para'} + If not 'none', use distinct models for subsets of instances: + * 'sent': intra- vs inter-sentential, + * 'sent-para': intra-sentential, intra-paragraph, rest (doc-level). + """ + # (re-)create a d-corpus from the RST-DT c-corpus + corpus_subset = os.path.basename(dataset).split('.')[0] + if corpus_subset not in ('TRAINING', 'TEST'): + raise ValueError("dataset must be a filepath that starts with" + "one of {'TRAINING', 'TEST'}") + if corpus_subset == 'TRAINING': + section = 'train' + else: # 'TEST' + section = 'test' + rst_ccorpus = read_corpus(corpus, section=section) + rel_conv = RstRelationConverter(RELMAP_112_18_FILE).convert_dtree + rst_dcorpus = dict() # FileId.doc -> RstDepTree + for doc_key, rst_ctree in rst_ccorpus[section].items(): + rst_dtree = RstDepTree.from_rst_tree(rst_ctree, nary_enc=nary_enc) + rst_dtree_coarse = rel_conv(rst_dtree) + rst_dcorpus[doc_key.doc] = rst_dtree_coarse + # for each candidate dependency in the dataset, read the nuclearity + # from the RST d-corpus + # Nota: we stream through the dataset to avoid loading it entirely in + # memory ; we don't need to open the vocabulary file (.vocab), nor the + # description of the EDUs (.edu_input) + pairings = dataset + '.pairings' + # edu_desc = dataset + '.edu_input' + if model_split == 'none': + new_dataset = os.path.join(out_dir, os.path.basename(dataset)) + new_pairs = os.path.join(out_dir, os.path.basename(pairings)) + if ((os.path.abspath(new_dataset) == os.path.abspath(dataset) or + os.path.abspath(new_pairs) == os.path.abspath(pairings))): + raise ValueError("I won't let you erase your base dataset") + with codecs.open(dataset, mode='rb', encoding='utf-8') as f_data: + with codecs.open(pairings, mode='rb', encoding='utf-8') as f_pairs: + with codecs.open(new_dataset, mode='wb', encoding='utf-8') as data_out: + with codecs.open(new_pairs, mode='wb', encoding='utf-8') as pairs_out: + # read header line in svmlight file + header = f_data.readline() + header_prefix = '# labels: ' + assert header.startswith(header_prefix) + labels = header[len(header_prefix):].split() + int2lbl = dict(enumerate(labels, start=1)) + lbl2int = {lbl: i for i, lbl in int2lbl.items()} + unrelated = lbl2int["UNRELATED"] + root = lbl2int["ROOT"] + # write labels in header of new svmlight file, as an + # ordered list mapped to {1, 2} + print(header_prefix + ' '.join((NUC_N, NUC_S)), + file=data_out) + # stream through lines + for pair, line in itertools.izip(f_pairs, f_data): + # read candidate pair of EDUs + src_id, tgt_id = pair.strip().split('\t') + if src_id == 'ROOT': + continue + # now both src_id and tgt_id are of form "docname_int" + # ex: "wsj_0600.out_1" + src_idx = int(src_id.rsplit('_', 1)[1]) + doc_name, tgt_idx = tgt_id.rsplit('_', 1) + tgt_idx = int(tgt_idx) + if tgt_idx < src_idx: + # skip left dependencies: by construction, + # their nuclearity can only be Satellite + # (SN edges) + continue + # print(doc_name, src_id, tgt_id, src_idx, tgt_idx) + # read corresponding ref class (label), feature vector + lbl_idx, feat_vector = line.strip().split(' ', 1) + lbl_idx = int(lbl_idx) # lbl currently encoded as int + if lbl_idx in (unrelated, root): + continue + try: + lbl = int2lbl[lbl_idx] + except KeyError: + # the test set in RST-DT 1.0 has an error: + # wsj_1189.out [8-9] is labelled "span" instead of + # "Consequence" ; some runs used this erroneous + # version, hence had a class "0" (unknown) for + # this line in the dataset + if ((doc_name == 'wsj_1189.out' and + src_idx == 7 and + tgt_idx == 9)): + lbl = 'cause' + lbl_idx = lbl2int[lbl] + else: + print(doc_name, src_idx, tgt_idx) + raise + # print(src_id, tgt_id, lbl) + dtree = rst_dcorpus[doc_name] + assert dtree.heads[tgt_idx] == src_idx + assert dtree.labels[tgt_idx] == lbl + if dtree.nucs[tgt_idx] == NUC_N: + nuc_idx = 1 + elif dtree.nucs[tgt_idx] == NUC_S: + nuc_idx = 2 + else: + raise ValueError("weird nuclearity {}".format( + dtree.nucs[tgt_idx])) + print(str(nuc_idx) + ' ' + feat_vector, + file=data_out) + print(pair.strip(), file=pairs_out) + elif model_split == 'sent': + # 2 datasets: intra- and inter-sentential + new_dataset = ( + os.path.join(out_dir + '_intrasent', os.path.basename(dataset)), + os.path.join(out_dir + '_intersent', os.path.basename(dataset)) + ) + new_pairs = ( + os.path.join(out_dir + '_intrasent', os.path.basename(pairings)), + os.path.join(out_dir + '_intersent', os.path.basename(pairings)) + ) + if ((os.path.abspath(new_dataset[0]) == os.path.abspath(dataset) or + os.path.abspath(new_pairs[0]) == os.path.abspath(pairings) or + os.path.abspath(new_dataset[1]) == os.path.abspath(dataset) or + os.path.abspath(new_pairs[1]) == os.path.abspath(pairings))): + raise ValueError("I won't let you erase your base dataset") + with codecs.open(dataset, mode='rb', encoding='utf-8') as f_data: + with codecs.open(pairings, mode='rb', encoding='utf-8') as f_pairs: + with codecs.open(new_dataset[0], mode='wb', encoding='utf-8') as data_out_intra: + with codecs.open(new_pairs[0], mode='wb', encoding='utf-8') as pairs_out_intra: + with codecs.open(new_dataset[1], mode='wb', encoding='utf-8') as data_out_inter: + with codecs.open(new_pairs[1], mode='wb', encoding='utf-8') as pairs_out_inter: + # read header line in svmlight file + header = f_data.readline() + header_prefix = '# labels: ' + assert header.startswith(header_prefix) + labels = header[len(header_prefix):].split() + int2lbl = dict(enumerate(labels, start=1)) + lbl2int = {lbl: i for i, lbl in int2lbl.items()} + unrelated = lbl2int["UNRELATED"] + root = lbl2int["ROOT"] + # write labels in header of new svmlight file, as an + # ordered list mapped to {1, 2} + print(header_prefix + ' '.join((NUC_N, NUC_S)), + file=data_out_intra) + print(header_prefix + ' '.join((NUC_N, NUC_S)), + file=data_out_inter) + # stream through lines + for pair, line in itertools.izip(f_pairs, f_data): + # read candidate pair of EDUs + src_id, tgt_id = pair.strip().split('\t') + if src_id == 'ROOT': + continue + # now both src_id and tgt_id are of form "docname_int" + # ex: "wsj_0600.out_1" + src_idx = int(src_id.rsplit('_', 1)[1]) + doc_name, tgt_idx = tgt_id.rsplit('_', 1) + tgt_idx = int(tgt_idx) + if tgt_idx < src_idx: + # skip left dependencies: by construction, + # their nuclearity can only be Satellite + # (SN edges) + continue + # print(doc_name, src_id, tgt_id, src_idx, tgt_idx) + # read corresponding ref class (label), feature vector + lbl_idx, feat_vector = line.strip().split(' ', 1) + lbl_idx = int(lbl_idx) # lbl currently encoded as int + if lbl_idx in (unrelated, root): + continue + try: + lbl = int2lbl[lbl_idx] + except KeyError: + # the test set in RST-DT 1.0 has an error: + # wsj_1189.out [8-9] is labelled "span" instead of + # "Consequence" ; some runs used this erroneous + # version, hence had a class "0" (unknown) for + # this line in the dataset + if ((doc_name == 'wsj_1189.out' and + src_idx == 7 and + tgt_idx == 9)): + lbl = 'cause' + lbl_idx = lbl2int[lbl] + else: + print(doc_name, src_idx, tgt_idx) + raise + # print(src_id, tgt_id, lbl) + dtree = rst_dcorpus[doc_name] + assert dtree.heads[tgt_idx] == src_idx + assert dtree.labels[tgt_idx] == lbl + if dtree.nucs[tgt_idx] == NUC_N: + nuc_idx = 1 + elif dtree.nucs[tgt_idx] == NUC_S: + nuc_idx = 2 + else: + raise ValueError("weird nuclearity {}".format( + dtree.nucs[tgt_idx])) + if ((' 269:' in feat_vector or + ' 303:' in feat_vector)): + # 269 is same_sentence_intra_right + # 303 is same_sentence_intra_left + # FIXME find a cleaner way + print(str(nuc_idx) + ' ' + feat_vector, + file=data_out_intra) + print(pair.strip(), + file=pairs_out_intra) + else: + # inter-sentential + print(str(nuc_idx) + ' ' + feat_vector, + file=data_out_inter) + print(pair.strip(), + file=pairs_out_inter) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser( + description='Prepare a nuclearity dataset.' + ) + parser.add_argument('--corpus', + help='Path to the RST-DT "main" corpus', + default=os.path.join( + os.path.expanduser('~'), + 'corpora/rst-dt/rst_discourse_treebank/data', + 'RSTtrees-WSJ-main-1.01' + )) + parser.add_argument('--dataset', + help='Base file of the dataset', + default=os.path.join( + os.path.expanduser('~'), + 'melodi/rst/irit-rst-dt/TMP/syn_pred_coarse', + 'TRAINING.relations.sparse' + )) + parser.add_argument('--out_dir', + help='Output folder', + default=os.path.join( + os.path.expanduser('~'), + 'melodi/rst/irit-rst-dt/TMP/syn_pred_coarse_NUC' + )) + parser.add_argument('--nary_enc', + help='Encoding for n-ary nodes', + choices=['chain', 'tree'], + default='chain') + parser.add_argument('--model_split', + help='Separate models for subsets of instances', + choices=['none', 'sent', 'sent-para'], + default='none') + args = parser.parse_args() + main(args.corpus, args.dataset, args.out_dir, args.nary_enc, + args.model_split) diff --git a/evals/prepare_rel_dataset.py b/evals/prepare_rel_dataset.py new file mode 100644 index 0000000..0bd6a5b --- /dev/null +++ b/evals/prepare_rel_dataset.py @@ -0,0 +1,256 @@ +"""This utility script outputs a dataset of the relation of RST edges. + +Given the path to the RST-DT corpus and a dataset of candidate RST +dependencies labelled with their gold coarse (class) RST relation (or +none if they are unrelated), produce a filtered version of the dataset +for the task of relation labelling. + +As of 2017-12-14, we filter out the instances for unrelated pairs of EDUs +and dependencies headed by the fake root. +The resulting dataset describes a n-ary classification problem whose +labelset is the set of (coarse-grained) classes of RST relations. +""" + +from __future__ import absolute_import, print_function + +import argparse +import codecs +import itertools +import os + +from educe.rst_dt.annotation import NUC_N, NUC_S +from educe.rst_dt.corpus import RstRelationConverter, RELMAP_112_18_FILE +from educe.rst_dt.dep_corpus import read_corpus +from educe.rst_dt.deptree import RstDepTree + + +def main(corpus, dataset, out_dir, nary_enc, model_split): + """Do prepare the RST relation dataset. + + Parameters + ---------- + corpus : str + Path to the RST-DT "main" corpus. + dataset : str + Path to the existing dataset labelled with coarse relations. + out_dir : str + Path to the output folder. + model_split : str, one of {'none', 'sent', 'sent-para'} + If not 'none', use distinct models for subsets of instances: + * 'sent': intra- vs inter-sentential, + * 'sent-para': intra-sentential, intra-paragraph, rest (doc-level). + """ + # (re-)create a d-corpus from the RST-DT c-corpus + corpus_subset = os.path.basename(dataset).split('.')[0] + if corpus_subset not in ('TRAINING', 'TEST'): + raise ValueError("dataset must be a filepath that starts with" + "one of {'TRAINING', 'TEST'}") + if corpus_subset == 'TRAINING': + section = 'train' + else: # 'TEST' + section = 'test' + rst_ccorpus = read_corpus(corpus, section=section) + rel_conv = RstRelationConverter(RELMAP_112_18_FILE).convert_dtree + rst_dcorpus = dict() # FileId.doc -> RstDepTree + for doc_key, rst_ctree in rst_ccorpus[section].items(): + rst_dtree = RstDepTree.from_rst_tree(rst_ctree, nary_enc=nary_enc) + rst_dtree_coarse = rel_conv(rst_dtree) + rst_dcorpus[doc_key.doc] = rst_dtree_coarse + # for each candidate dependency in the dataset, read the nuclearity + # from the RST d-corpus + # Nota: we stream through the dataset to avoid loading it entirely in + # memory ; we don't need to open the vocabulary file (.vocab), nor the + # description of the EDUs (.edu_input) + pairings = dataset + '.pairings' + # edu_desc = dataset + '.edu_input' + if model_split == 'none': + new_dataset = os.path.join(out_dir, os.path.basename(dataset)) + new_pairs = os.path.join(out_dir, os.path.basename(pairings)) + if ((os.path.abspath(new_dataset) == os.path.abspath(dataset) or + os.path.abspath(new_pairs) == os.path.abspath(pairings))): + raise ValueError("I won't let you erase your base dataset") + with codecs.open(dataset, mode='rb', encoding='utf-8') as f_data: + with codecs.open(pairings, mode='rb', encoding='utf-8') as f_pairs: + with codecs.open(new_dataset, mode='wb', encoding='utf-8') as data_out: + with codecs.open(new_pairs, mode='wb', encoding='utf-8') as pairs_out: + # read header line in svmlight file + header = f_data.readline() + header_prefix = '# labels: ' + assert header.startswith(header_prefix) + labels = header[len(header_prefix):].split() + int2lbl = dict(enumerate(labels, start=1)) + lbl2int = {lbl: i for i, lbl in int2lbl.items()} + unrelated = lbl2int["UNRELATED"] + root = lbl2int["ROOT"] + # write labels in header of new svmlight file, here + # we just copy the existing header (even if it has + # ROOT and UNRELATED that should never appear here) + print(header, file=data_out) + # stream through lines + for pair, line in itertools.izip(f_pairs, f_data): + # read candidate pair of EDUs + src_id, tgt_id = pair.strip().split('\t') + if src_id == 'ROOT': + continue + # now both src_id and tgt_id are of form "docname_int" + # ex: "wsj_0600.out_1" + src_idx = int(src_id.rsplit('_', 1)[1]) + doc_name, tgt_idx = tgt_id.rsplit('_', 1) + tgt_idx = int(tgt_idx) + # read corresponding ref class (label), feature vector + lbl_idx, feat_vector = line.strip().split(' ', 1) + lbl_idx = int(lbl_idx) # lbl currently encoded as int + if lbl_idx in (unrelated, root): + continue + try: + lbl = int2lbl[lbl_idx] + except KeyError: + # the test set in RST-DT 1.0 has an error: + # wsj_1189.out [8-9] is labelled "span" instead of + # "Consequence" ; some runs used this erroneous + # version, hence had a class "0" (unknown) for + # this line in the dataset + if ((doc_name == 'wsj_1189.out' and + src_idx == 7 and + tgt_idx == 9)): + lbl = 'cause' + lbl_idx = lbl2int[lbl] + else: + print(doc_name, src_idx, tgt_idx) + raise + # print(src_id, tgt_id, lbl) + dtree = rst_dcorpus[doc_name] + assert dtree.heads[tgt_idx] == src_idx + assert dtree.labels[tgt_idx] == lbl + print(str(lbl_idx) + ' ' + feat_vector, + file=data_out) + print(pair.strip(), file=pairs_out) + elif model_split == 'sent': + # 2 datasets: intra- and inter-sentential + new_dataset = ( + os.path.join(out_dir + '_intrasent', os.path.basename(dataset)), + os.path.join(out_dir + '_intersent', os.path.basename(dataset)) + ) + new_pairs = ( + os.path.join(out_dir + '_intrasent', os.path.basename(pairings)), + os.path.join(out_dir + '_intersent', os.path.basename(pairings)) + ) + if ((os.path.abspath(new_dataset[0]) == os.path.abspath(dataset) or + os.path.abspath(new_pairs[0]) == os.path.abspath(pairings) or + os.path.abspath(new_dataset[1]) == os.path.abspath(dataset) or + os.path.abspath(new_pairs[1]) == os.path.abspath(pairings))): + raise ValueError("I won't let you erase your base dataset") + with codecs.open(dataset, mode='rb', encoding='utf-8') as f_data: + with codecs.open(pairings, mode='rb', encoding='utf-8') as f_pairs: + with codecs.open(new_dataset[0], mode='wb', encoding='utf-8') as data_out_intra: + with codecs.open(new_pairs[0], mode='wb', encoding='utf-8') as pairs_out_intra: + with codecs.open(new_dataset[1], mode='wb', encoding='utf-8') as data_out_inter: + with codecs.open(new_pairs[1], mode='wb', encoding='utf-8') as pairs_out_inter: + # read header line in svmlight file + header = f_data.readline() + header_prefix = '# labels: ' + assert header.startswith(header_prefix) + labels = header[len(header_prefix):].split() + int2lbl = dict(enumerate(labels, start=1)) + lbl2int = {lbl: i for i, lbl in int2lbl.items()} + unrelated = lbl2int["UNRELATED"] + root = lbl2int["ROOT"] + # write labels in header of new svmlight file + print(header, file=data_out_intra) + print(header, file=data_out_inter) + # stream through lines + for pair, line in itertools.izip(f_pairs, f_data): + # read candidate pair of EDUs + src_id, tgt_id = pair.strip().split('\t') + if src_id == 'ROOT': + continue + # now both src_id and tgt_id are of form "docname_int" + # ex: "wsj_0600.out_1" + src_idx = int(src_id.rsplit('_', 1)[1]) + doc_name, tgt_idx = tgt_id.rsplit('_', 1) + tgt_idx = int(tgt_idx) + # read corresponding ref class (label), feature vector + lbl_idx, feat_vector = line.strip().split(' ', 1) + lbl_idx = int(lbl_idx) # lbl currently encoded as int + if lbl_idx in (unrelated, root): + continue + try: + lbl = int2lbl[lbl_idx] + except KeyError: + # the test set in RST-DT 1.0 has an error: + # wsj_1189.out [8-9] is labelled "span" instead of + # "Consequence" ; some runs used this erroneous + # version, hence had a class "0" (unknown) for + # this line in the dataset + if ((doc_name == 'wsj_1189.out' and + src_idx == 7 and + tgt_idx == 9)): + lbl = 'cause' + lbl_idx = lbl2int[lbl] + else: + print(doc_name, src_idx, tgt_idx) + raise + # print(src_id, tgt_id, lbl) + dtree = rst_dcorpus[doc_name] + assert dtree.heads[tgt_idx] == src_idx + assert dtree.labels[tgt_idx] == lbl + if ((' 269:' in feat_vector or + ' 303:' in feat_vector) and + (' 103:' in feat_vector or + ' 158:' in feat_vector or + ' 234:' in feat_vector or + ' 314:' in feat_vector)): + # 269 is same_sentence_intra_right + # 303 is same_sentence_intra_left ; + # 103 is same_para_inter_right + # 158 is same_para_inter_left + # 234 is same_para_intra_right + # 314 is same_para_intra_left + # FIXME find a cleaner way + print(str(lbl_idx) + ' ' + feat_vector, + file=data_out_intra) + print(pair.strip(), + file=pairs_out_intra) + else: + # inter-sentential + print(str(lbl_idx) + ' ' + feat_vector, + file=data_out_inter) + print(pair.strip(), + file=pairs_out_inter) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser( + description='Prepare a relation dataset.' + ) + parser.add_argument('--corpus', + help='Path to the RST-DT "main" corpus', + default=os.path.join( + os.path.expanduser('~'), + 'corpora/rst-dt/rst_discourse_treebank/data', + 'RSTtrees-WSJ-main-1.01' + )) + parser.add_argument('--dataset', + help='Base file of the dataset', + default=os.path.join( + os.path.expanduser('~'), + 'melodi/rst/irit-rst-dt/TMP/syn_pred_coarse', + 'TRAINING.relations.sparse' + )) + parser.add_argument('--out_dir', + help='Output folder', + default=os.path.join( + os.path.expanduser('~'), + 'melodi/rst/irit-rst-dt/TMP/syn_pred_coarse_REL' + )) + parser.add_argument('--nary_enc', + help='Encoding for n-ary nodes', + choices=['chain', 'tree'], + default='chain') + parser.add_argument('--model_split', + help='Separate models for subsets of instances', + choices=['none', 'sent', 'sent-para'], + default='none') + args = parser.parse_args() + main(args.corpus, args.dataset, args.out_dir, args.nary_enc, + args.model_split) diff --git a/evals/showdown.py b/evals/showdown.py new file mode 100644 index 0000000..1fe101a --- /dev/null +++ b/evals/showdown.py @@ -0,0 +1,949 @@ +"""This module evaluates the output of discourse parsers. + +Included are dependency and constituency tree metrics. +""" + +from __future__ import print_function + +import argparse +import codecs +import itertools +import os + +from sklearn.datasets import load_svmlight_files +from sklearn.linear_model.logistic import LogisticRegression, LogisticRegressionCV + +from educe.rst_dt.annotation import _binarize, SimpleRSTTree +from educe.rst_dt.corpus import (RstRelationConverter, + Reader as RstReader) +from educe.rst_dt.dep2con import (DummyNuclearityClassifier, + InsideOutAttachmentRanker) +from educe.rst_dt.deptree import RstDepTree +from educe.rst_dt.metrics.rst_parseval import (rst_parseval_detailed_report, + rst_parseval_compact_report, + rst_parseval_report, + rst_parseval_similarity) +# +from attelo.metrics.deptree import (compute_uas_las, + dep_compact_report, + dep_similarity) + +# local to this package +from evals.braud_coling import (load_braud_coling_ctrees, + load_braud_coling_dtrees) +from evals.braud_eacl import (load_braud_eacl_ctrees, + load_braud_eacl_dtrees) +from evals.codra import load_codra_ctrees, load_codra_dtrees +from evals.feng import load_feng_ctrees, load_feng_dtrees +from evals.gcrf_tree_format import load_gcrf_ctrees, load_gcrf_dtrees +from evals.hayashi_cons import (load_hayashi_hilda_ctrees, + load_hayashi_hilda_dtrees) +from evals.hayashi_deps import (load_hayashi_dep_dtrees, + load_hayashi_dep_ctrees) +from evals.ji import load_ji_ctrees, load_ji_dtrees +from evals.li_qi import load_li_qi_ctrees, load_li_qi_dtrees +from evals.li_sujian import (DEFAULT_FILE as LI_SUJIAN_OUT_FILE, + load_li_sujian_dep_ctrees, + load_li_sujian_dep_dtrees) +from evals.ours import (load_deptrees_from_attelo_output, + load_attelo_ctrees, + load_attelo_dtrees) +from evals.surdeanu import load_surdeanu_ctrees, load_surdeanu_dtrees +# 2017-12-12 nuc_clf WIP +from evals.train_nuc_classifier import RightBinaryNuclearityClassifier +from evals.train_rel_relabeller import RelationRelabeller +# end WIP nuc_clf + +# RST corpus +CORPUS_DIR = os.path.join('corpus', 'RSTtrees-WSJ-main-1.01/') +CD_TRAIN = os.path.join(CORPUS_DIR, 'TRAINING') +CD_TEST = os.path.join(CORPUS_DIR, 'TEST') +DOUBLE_DIR = os.path.join('corpus', 'RSTtrees-WSJ-double-1.0') +# relation converter (fine- to coarse-grained labels) +RELMAP_FILE = os.path.join('/home/mmorey/melodi/educe', + 'educe', 'rst_dt', + 'rst_112to18.txt') +REL_CONV_BASE = RstRelationConverter(RELMAP_FILE) +REL_CONV = REL_CONV_BASE.convert_tree +REL_CONV_DTREE = REL_CONV_BASE.convert_dtree + + +# +# EVALUATIONS +# + +# * syntax: pred vs gold +# old-style .edu_input: whole test set +EDUS_FILE = os.path.join('/home/mmorey', + 'melodi/rst', + 'irit-rst-dt/TMP/syn_gold_coarse', + 'TEST.relations.sparse.edu_input') + +# new style .edu_input: one file per doc in test set +# was: TMP/latest/data..., replaced latest with 2016-09-30T1701 but +# might be wrong (or it might have no consequence here) +EDUS_FILE_PAT = "TMP/2016-09-30T1701/data/TEST/{}.relations.edu-pairs.sparse.edu_input" + +# outputs of parsers +EISNER_OUT_SYN_PRED = os.path.join( + '/home/mmorey', + 'melodi/rst', + 'irit-rst-dt/TMP/syn_pred_coarse', # lbl + 'scratch-current/combined', + 'output.maxent-iheads-global-AD.L-jnt-eisner') + +# 2016-09-14 "tree" transform, predicted syntax +EISNER_OUT_TREE_SYN_PRED = os.path.join( + '/home/mmorey', + 'melodi/rst', + 'irit-rst-dt/TMP/2016-09-12T0825', # lbl + 'scratch-current/combined', + 'output.maxent-iheads-global-AD.L-jnt-eisner') + +EISNER_OUT_TREE_SYN_PRED_SU = os.path.join( + '/home/mmorey', + 'melodi/rst', + 'irit-rst-dt/TMP/2016-09-12T0825', # lbl + 'scratch-current/combined', + 'output.maxent-iheads-global-AD.L-jnt_su-eisner') +# end 2016-09-14 + + +EISNER_OUT_SYN_PRED_SU = os.path.join( + '/home/mmorey', + 'melodi/rst', + 'irit-rst-dt/TMP/latest', # lbl + 'scratch-current/combined', + 'output.maxent-AD.L-jnt_su-eisner') + +EISNER_OUT_SYN_GOLD = os.path.join( + '/home/mmorey', + 'melodi/rst', + 'irit-rst-dt/TMP/syn_gold_coarse', # lbl + 'scratch-current/combined', + 'output.maxent-iheads-global-AD.L-jnt-eisner') + +# output of Joty's parser CODRA +CODRA_OUT_DIR = os.path.join( + '/home/mmorey', + 'melodi/rst/replication/joty/Doc-level' +) +# output of Ji's parser DPLP +# JI_OUT_DIR = os.path.join('/home/mmorey/melodi/rst/replication/ji_eisenstein', 'DPLP/data/docs/test/') +JI_OUT_DIR = os.path.join('/home/mmorey', + 'melodi/rst/replication/ji_eisenstein', + 'official_output/outputs/') +# Feng's parsers +FENG_DIR = os.path.join('/home/mmorey', + 'melodi/rst/replication/feng_hirst/') +FENG1_OUT_DIR = os.path.join(FENG_DIR, 'phil', 'tmp') +FENG2_OUT_DIR = os.path.join(FENG_DIR, 'gCRF_dist/texts/results/test_batch_gold_seg') +# Li Qi's parser +LI_QI_OUT_DIR = os.path.join('/home/mmorey', + 'melodi/rst/replication/li_qi/result') +# Hayashi's HILDA +HAYASHI_OUT_DIR = '/home/mmorey/melodi/rst/replication/hayashi/SIGDIAL' +HAYASHI_HILDA_OUT_DIR = os.path.join(HAYASHI_OUT_DIR, 'auto_parse/cons/HILDA') +HAYASHI_MST_OUT_DIR = os.path.join(HAYASHI_OUT_DIR, 'auto_parse/dep/li') +# Braud +BRAUD_COLING_OUT_DIR = '/home/mmorey/melodi/rst/replication/braud/coling16/pred_trees' +BRAUD_EACL_MONO = '/home/mmorey/melodi/rst/replication/braud/eacl16/best-en-mono/test_it8_beam16' +BRAUD_EACL_CROSS_DEV = '/home/mmorey/melodi/rst/replication/braud/eacl16/best-en-cross+dev/test_it10_beam32' +# Surdeanu +SURDEANU_LOG_FILE = '/home/mmorey/melodi/rst/replication/surdeanu/output/log' +# Li Sujian dep parser +# imported, see above +# Wang, Li and Wang at ACL 2017 +WLW17_OUT_DIR = os.path.join( + '/home/mmorey', + 'melodi/rst/replication/wang/rst-dt/RSTtrees-WSJ-main-1.0/TEST') + +# level of detail for parseval +STRINGENT = False +# additional dependency metrics +INCLUDE_LS = False +EVAL_NUC_RANK = True +# hyperparams +NUC_STRATEGY = 'unamb_else_most_frequent' +NUC_CONSTANT = None # only useful for NUC_STRATEGY='constant' +RNK_STRATEGY = 'sdist-edist-rl' +RNK_PRIORITY_SU = True +# known 'authors' +AUTHORS = [ + 'gold', # RST-main + 'silver', # RST-double + 'JCN15_1S1S', 'FH14_gSVM', 'FH14_gCRF', 'JE14', + 'LLC16', 'HHN16_HILDA', 'HHN16_MST', + 'BPS16', 'BCS17_mono', + 'BCS17_cross', + 'SHV15_D', + 'WLW17', # Wang, Li and Wang, ACL17 + 'li_sujian', + 'ours-chain', 'ours-tree', 'ours-tree-su' +] + + +def setup_dtree_postprocessor(nary_enc='chain', order='strict', + nuc_strategy=NUC_STRATEGY, + nuc_constant=NUC_CONSTANT, + rnk_strategy=RNK_STRATEGY, + rnk_prioritize_same_unit=RNK_PRIORITY_SU): + """Setup the nuclearity and rank classifiers to flesh out dtrees.""" + # load train section of the RST corpus, fit (currently dummy) classifiers + # for nuclearity and rank + reader_train = RstReader(CD_TRAIN) + corpus_train = reader_train.slurp() + # gold RST trees + ctree_true = dict() # ctrees + dtree_true = dict() # dtrees from the original ctrees ('tree' transform) + + for doc_id, ct_true in sorted(corpus_train.items()): + doc_name = doc_id.doc + # flavours of ctree + ct_true = REL_CONV(ct_true) # map fine to coarse relations + ctree_true[doc_name] = ct_true + # flavours of dtree + dt_true = RstDepTree.from_rst_tree(ct_true, nary_enc=nary_enc) + dtree_true[doc_name] = dt_true + # fit classifiers for nuclearity and rank (DIRTY) + # NB: both are (dummily) fit on weakly ordered dtrees + X_train = [] + y_nuc_train = [] + y_rnk_train = [] + for doc_name, dt in sorted(dtree_true.items()): + # print(dt.__dict__) + # raise ValueError('wip wip nuc_clf') + X_train.append(dt) + y_nuc_train.append(dt.nucs) + y_rnk_train.append(dt.ranks) + # 2017-12-14 WIP relation relabeller + if False: + model_split = 'sent' # {'none', 'sent'} + if model_split == 'none': + dset_folder = os.path.join( + os.path.expanduser('~'), + 'melodi/rst/irit-rst-dt/TMP/syn_pred_coarse_REL' + ) + dset_rel_train = os.path.join(dset_folder, 'TRAINING.relations.sparse') + dset_rel_test = os.path.join(dset_folder, 'TEST.relations.sparse') + # FIXME read n_features from .vocab + X_rel_train, y_rel_train, X_rel_test, y_rel_test = load_svmlight_files( + (dset_rel_train, dset_rel_test), + n_features=46731, + zero_based=False + ) + elif model_split == 'sent': + # * intra + dset_folder_intra = os.path.join( + os.path.expanduser('~'), + 'melodi/rst/irit-rst-dt/TMP/syn_pred_coarse_REL_intrasent' + ) + dset_train_intra = os.path.join(dset_folder_intra, 'TRAINING.relations.sparse') + dset_test_intra = os.path.join(dset_folder_intra, 'TEST.relations.sparse') + # FIXME read n_features from .vocab + X_rel_train_intra, y_rel_train_intra, X_rel_test_intra, y_rel_test_intra = load_svmlight_files( + (dset_train_intra, dset_test_intra), + n_features=46731, + zero_based=False + ) + # * inter + dset_folder_inter = os.path.join( + os.path.expanduser('~'), + 'melodi/rst/irit-rst-dt/TMP/syn_pred_coarse_REL_intersent' + ) + dset_train_inter = os.path.join(dset_folder_inter, 'TRAINING.relations.sparse') + dset_test_inter = os.path.join(dset_folder_inter, 'TEST.relations.sparse') + # FIXME read n_features from .vocab + X_rel_train_inter, y_rel_train_inter, X_rel_test_inter, y_rel_test_inter = load_svmlight_files( + (dset_train_inter, dset_test_inter), + n_features=46731, + zero_based=False + ) + # put together intra and inter + X_rel_train = (X_rel_train_intra, X_rel_train_inter) + y_rel_train = (y_rel_train_intra, y_rel_train_inter) + # TODO the same for {X,y}_rel_test ? + else: + raise ValueError("what model_split?") + # common call + mul_clf = LogisticRegressionCV(Cs=10, # defaults to 10, + penalty='l1', solver='liblinear', + n_jobs=3) + rel_clf = RelationRelabeller(mul_clf=mul_clf, model_split=model_split) + rel_clf = rel_clf.fit(X_rel_train, y_rel_train) + else: + rel_clf = None + # end 2017-12-14 relations relabeller + # nuclearity clf + if True: + # TODO see whether intra/inter-sentential would be good + # for the dummy nuc clf + nuc_clf = DummyNuclearityClassifier(strategy=nuc_strategy, + constant=nuc_constant) + nuc_clf.fit(X_train, y_nuc_train) + else: + # 2017-12-12 WIP nuc_clf + # shiny new nuc_clf ; still very hacky + # import the nuclearity TRAIN and TEST sets generated from + # the svmlight feature vectors (ahem) + model_split = 'sent' + # + if model_split == 'none': + dset_folder = os.path.join( + os.path.expanduser('~'), + 'melodi/rst/irit-rst-dt/TMP/syn_pred_coarse_NUC' + ) + dset_train = os.path.join(dset_folder, 'TRAINING.relations.sparse') + dset_test = os.path.join(dset_folder, 'TEST.relations.sparse') + # FIXME read n_features from .vocab + X_nuc_train, y_nuc_train, X_nuc_test, y_nuc_test = load_svmlight_files( + (dset_train, dset_test), + n_features=46731, + zero_based=False + ) + elif model_split == 'sent': + # * intra + dset_folder_intra = os.path.join( + os.path.expanduser('~'), + 'melodi/rst/irit-rst-dt/TMP/syn_pred_coarse_NUC_intrasent' + ) + dset_train_intra = os.path.join(dset_folder_intra, 'TRAINING.relations.sparse') + dset_test_intra = os.path.join(dset_folder_intra, 'TEST.relations.sparse') + # FIXME read n_features from .vocab + X_nuc_train_intra, y_nuc_train_intra, X_nuc_test_intra, y_nuc_test_intra = load_svmlight_files( + (dset_train_intra, dset_test_intra), + n_features=46731, + zero_based=False + ) + # * inter + dset_folder_inter = os.path.join( + os.path.expanduser('~'), + 'melodi/rst/irit-rst-dt/TMP/syn_pred_coarse_NUC_intersent' + ) + dset_train_inter = os.path.join(dset_folder_inter, 'TRAINING.relations.sparse') + dset_test_inter = os.path.join(dset_folder_inter, 'TEST.relations.sparse') + # FIXME read n_features from .vocab + X_nuc_train_inter, y_nuc_train_inter, X_nuc_test_inter, y_nuc_test_inter = load_svmlight_files( + (dset_train_inter, dset_test_inter), + n_features=46731, + zero_based=False + ) + # put together intra and inter + X_nuc_train = (X_nuc_train_intra, X_nuc_train_inter) + y_nuc_train = (y_nuc_train_intra, y_nuc_train_inter) + # TODO the same for {X,y}_nuc_test ? + else: + raise ValueError("what model_split?") + bin_clf = LogisticRegressionCV(Cs=10, # defaults to 10 + penalty='l1', solver='liblinear', + n_jobs=3) + nuc_clf = RightBinaryNuclearityClassifier(bin_clf=bin_clf, + model_split=model_split) + nuc_clf = nuc_clf.fit(X_nuc_train, y_nuc_train) + # end WIP nuc_clf + # rank clf + rnk_clf = InsideOutAttachmentRanker( + strategy=rnk_strategy, prioritize_same_unit=rnk_prioritize_same_unit, + order=order) + rnk_clf.fit(X_train, y_rnk_train) + return nuc_clf, rnk_clf, rel_clf + + +# FIXME: +# * [ ] create summary table with one system per row, one metric per column, +# keep only the f-score (because for binary trees with manual segmentation +# precision = recall = f-score). +def main(): + """Run the eval""" + parser = argparse.ArgumentParser( + description="Evaluate parsers' output against a given reference") + # predictions + parser.add_argument('authors_pred', nargs='+', + choices=AUTHORS, + help="Author(s) of the predictions") + # reference + parser.add_argument('--author_true', default='gold', + choices=AUTHORS + ['each'], # NEW generate sim matrix + help="Author of the reference") + # * ctree/dtree eval: the value of binarize_true determines the values + # of nary_enc_true and order_true (the latter is yet unused) + parser.add_argument('--binarize_true', default='none', + choices=['none', 'right', 'right_mixed', 'left'], + help=("Binarization method for the reference ctree" + "in the eval ; defaults to 'none' for no " + "binarization")) + parser.add_argument('--simple_rsttree', action='store_true', + help="Binarize ctree and move relations up") + # * non-standard evals + parser.add_argument('--per_doc', action='store_true', + help="Doc-averaged scores (cf. Ji's eval)") + parser.add_argument('--eval_li_dep', action='store_true', + help=("Evaluate as in the dep parser of Li et al. " + "2014: all relations are NS, spiders map to " + "left-heavy branching, three trivial spans ")) + # * display options + parser.add_argument('--digits', type=int, default=3, + help='Precision (number of digits) of scores') + parser.add_argument('--percent', action='store_true', + help='Scores are displayed as percentages (ex: 57.9)') + parser.add_argument('--detailed', type=int, default=0, + help='Level of detail for evaluations') + parser.add_argument('--out_fmt', default='text', + choices=['text', 'latex'], + help='Output format') + # + args = parser.parse_args() + author_true = args.author_true + authors_pred = args.authors_pred + binarize_true = args.binarize_true + simple_rsttree = args.simple_rsttree + # display + digits = args.digits + percent = args.percent + if percent: + if digits < 3: + raise ValueError('--percent requires --digits >= 3') + # level of detail for evals + detailed = args.detailed + out_fmt = args.out_fmt + + # "per_doc = True" computes p, r, f as in DPLP: compute scores per doc + # then average over docs + # it should be False, except for comparison with the DPLP paper + per_doc = args.per_doc + # "eval_li_dep = True" replaces the original nuclearity and order with + # heuristically determined values for _pred but also _true, and adds + # three trivial spans + eval_li_dep = args.eval_li_dep + # nary_enc_true is used ; order_true currently is not (implicit in + # nary_enc_true) + if binarize_true in ('right', 'right_mixed'): + nary_enc_true = 'chain' + order_true = 'strict' + elif binarize_true == 'left': + nary_enc_true = 'tree' + order_true = 'strict' + else: # 'none' for no binarization of the reference tree + nary_enc_true = 'tree' + order_true = 'weak' + + # 0. setup the postprocessors to flesh out unordered dtrees into ordered + # ones with nuclearity + # * tie the order with the encoding for n-ary nodes + nuc_clf_chain, rnk_clf_chain, rel_clf_chain = setup_dtree_postprocessor( + nary_enc='chain', order='strict') + # FIXME explicit differenciation between (heuristic) classifiers for + # the "chain" vs "tree" transforms (2 parameters: nary_enc, order) ; + # nuc_clf, rnk_clf, rel_clf might contain implicit assumptions + # tied to the "chain" transform, might not be optimal for "tree" + nuc_clf_tree, rnk_clf_tree, rel_clf_tree = setup_dtree_postprocessor( + nary_enc='tree', order='weak') + + # the eval compares parses for the test section of the RST corpus + reader_test = RstReader(CD_TEST) + corpus_test = reader_test.slurp() + doc_edus_test = {k.doc: ct_true.leaves() for k, ct_true + in corpus_test.items()} + + # reference: author_true can be any of the authors_pred (defaults to gold) + ctree_true = dict() # ctrees + dtree_true = dict() # dtrees from the original ctrees ('tree' transform) + for doc_id, ct_true in sorted(corpus_test.items()): + doc_name = doc_id.doc + # original reference ctree, with coarse labels + ct_true = REL_CONV(ct_true) # map fine to coarse relations + if binarize_true != "none": + # binarize ctree if required + ct_true = _binarize(ct_true, branching=binarize_true) + ctree_true[doc_name] = ct_true + # corresponding dtree + dt_true = RstDepTree.from_rst_tree(ct_true, nary_enc=nary_enc_true) + dtree_true[doc_name] = dt_true + # sorted doc_names, because braud_eacl put all predictions in one file + sorted_doc_names = sorted(dtree_true.keys()) + + c_preds = [] # predictions: [(parser_name, dict(doc_name, ct_pred))] + d_preds = [] # predictions: [(parser_name, dict(doc_name, dt_pred))] + + for author_pred in authors_pred: + # braud coling 2016 + if author_pred == 'BPS16': + ctree_pred = load_braud_coling_ctrees(BRAUD_COLING_OUT_DIR, + REL_CONV) + c_preds.append( + ('BPS16', ctree_pred) + ) + d_preds.append( + ('BPS16', load_braud_coling_dtrees( + BRAUD_COLING_OUT_DIR, REL_CONV, nary_enc='chain', + ctree_pred=ctree_pred)) + ) + # braud eacl 2017 - mono + if author_pred == 'BCS17_mono': + ctree_pred = load_braud_eacl_ctrees(BRAUD_EACL_MONO, REL_CONV, + sorted_doc_names) + c_preds.append( + ('BCS17_mono', ctree_pred) + ) + d_preds.append( + ('BCS17_mono', load_braud_eacl_dtrees( + BRAUD_EACL_MONO, REL_CONV, sorted_doc_names, + nary_enc='chain', ctree_pred=ctree_pred)) + ) + # braud eacl 2017 - cross+dev + if author_pred == 'BCS17_cross': + ctree_pred = load_braud_eacl_ctrees(BRAUD_EACL_CROSS_DEV, + REL_CONV, sorted_doc_names) + c_preds.append( + ('BCS17_cross', ctree_pred) + ) + d_preds.append( + ('BCS17_cross', load_braud_eacl_dtrees( + BRAUD_EACL_CROSS_DEV, REL_CONV, sorted_doc_names, + nary_enc='chain', ctree_pred=ctree_pred)) + ) + + if author_pred == 'HHN16_HILDA': + ctree_pred = load_hayashi_hilda_ctrees(HAYASHI_HILDA_OUT_DIR, + REL_CONV) + c_preds.append( + ('HHN16_HILDA', ctree_pred) + ) + d_preds.append( + ('HHN16_HILDA', load_hayashi_hilda_dtrees( + HAYASHI_HILDA_OUT_DIR, REL_CONV, nary_enc='chain', + ctree_pred=ctree_pred)) + ) + + if author_pred == 'HHN16_MST': + # paper: {nary_enc_pred='chain', order='strict'} + dtree_pred = load_hayashi_dep_dtrees( + HAYASHI_MST_OUT_DIR, REL_CONV_DTREE, doc_edus_test, + EDUS_FILE_PAT, nuc_clf_chain, rnk_clf_chain) + c_preds.append( + ('HHN16_MST', load_hayashi_dep_ctrees( + HAYASHI_MST_OUT_DIR, REL_CONV_DTREE, doc_edus_test, + EDUS_FILE_PAT, nuc_clf_chain, rnk_clf_chain, + dtree_pred=dtree_pred)) + ) + d_preds.append( + ('HHN16_MST', dtree_pred) + ) + + if author_pred == 'LLC16': + ctree_pred = load_li_qi_ctrees(LI_QI_OUT_DIR, REL_CONV) + c_preds.append( + ('LLC16', ctree_pred) + ) + d_preds.append( + ('LLC16', load_li_qi_dtrees(LI_QI_OUT_DIR, REL_CONV, + nary_enc='chain', + ctree_pred=ctree_pred)) + ) + + if author_pred == 'li_sujian': + # FIXME load d-trees once, pass dtree_pred to the c-loader ; + # paper says 'chain' transform, but it might be worth + # checking + c_preds.append( + ('li_sujian', load_li_sujian_dep_ctrees( + LI_SUJIAN_OUT_FILE, REL_CONV_DTREE, EDUS_FILE_PAT, + nuc_clf_chain, rnk_clf_chain)) + ) + d_preds.append( + ('li_sujian', load_li_sujian_dep_dtrees( + LI_SUJIAN_OUT_FILE, REL_CONV_DTREE, EDUS_FILE_PAT, + nuc_clf_chain, rnk_clf_chain)) + ) + + if author_pred == 'FH14_gSVM': + # FIXME load c-trees once, pass ctree_pred to the d-loader + c_preds.append( + ('FH14_gSVM', load_feng_ctrees(FENG1_OUT_DIR, REL_CONV)) + ) + d_preds.append( + ('FH14_gSVM', load_feng_dtrees(FENG1_OUT_DIR, REL_CONV, + nary_enc='chain')) + ) + + if author_pred == 'FH14_gCRF': + ctree_pred = load_gcrf_ctrees(FENG2_OUT_DIR, REL_CONV) + c_preds.append( + ('FH14_gCRF', ctree_pred) + ) + d_preds.append( + ('FH14_gCRF', load_gcrf_dtrees(FENG2_OUT_DIR, REL_CONV, + nary_enc='chain', + ctree_pred=ctree_pred)) + ) + + if author_pred == 'JCN15_1S1S': + # CODRA outputs RST ctrees ; eval_codra_output maps them to RST dtrees + ctree_pred = load_codra_ctrees(CODRA_OUT_DIR, REL_CONV) + c_preds.append( + ('JCN15_1S1S', ctree_pred) + ) + d_preds.append( + ('JCN15_1S1S', load_codra_dtrees(CODRA_OUT_DIR, REL_CONV, + nary_enc='chain', + ctree_pred=ctree_pred)) + ) + # joty-{chain,tree} would be the same except nary_enc='tree' ; + # the nary_enc does not matter because codra outputs binary ctrees, + # hence both encodings result in (the same) strictly ordered dtrees + + if author_pred == 'JE14': + # DPLP outputs RST ctrees in the form of lists of spans; + # load_ji_dtrees maps them to RST dtrees + ctree_pred = load_ji_ctrees(JI_OUT_DIR, REL_CONV, doc_edus_test) + c_preds.append( + ('JE14', ctree_pred) + ) + d_preds.append( + ('JE14', load_ji_dtrees(JI_OUT_DIR, REL_CONV, doc_edus_test, + nary_enc='chain', + ctree_pred=ctree_pred)) + ) + # ji-{chain,tree} would be the same except nary_enc='tree' ; + # the nary_enc does not matter because DPLP outputs binary ctrees, + # hence both encodings result in (the same) strictly ordered dtrees + + if author_pred == 'WLW17': + # WLW17 outputs RST ctrees in the form of lists of spans, just + # like JE14 ; + # load_ji_dtrees maps them to RST dtrees + c_preds.append( + ('WLW17', load_ji_ctrees( + WLW17_OUT_DIR, REL_CONV)) + ) + d_preds.append( + ('WLW17', load_ji_dtrees( + WLW17_OUT_DIR, REL_CONV, nary_enc='chain')) + ) + # the nary_enc does not matter because WLW17 outputs binary ctrees, + # hence both encodings result in (the same) strictly ordered dtrees + + if author_pred == 'SHV15_D': + ctree_pred = load_surdeanu_ctrees(SURDEANU_LOG_FILE, REL_CONV) + c_preds.append( + ('SHV15_D', ctree_pred) + ) + d_preds.append( + ('SHV15_D', load_surdeanu_dtrees( + SURDEANU_LOG_FILE, REL_CONV, nary_enc='chain', + ctree_pred=ctree_pred)) + ) + + if author_pred == 'ours-chain': + # Eisner, predicted syntax, chain + dtree_pred = load_attelo_dtrees( + EISNER_OUT_SYN_PRED, EDUS_FILE, + rel_clf_chain, nuc_clf_chain, rnk_clf_chain, + doc_edus=doc_edus_test) + c_preds.append( + ('ours-chain', load_attelo_ctrees( + EISNER_OUT_SYN_PRED, EDUS_FILE, + rel_clf_chain, nuc_clf_chain, rnk_clf_chain, + doc_edus=doc_edus_test, + dtree_pred=dtree_pred)) + ) + d_preds.append( + ('ours-chain', dtree_pred) + ) + + if author_pred == 'ours-tree': + # Eisner, predicted syntax, tree + same-unit + dtree_pred = load_attelo_dtrees( + EISNER_OUT_TREE_SYN_PRED, EDUS_FILE, + rel_clf_tree, nuc_clf_tree, rnk_clf_tree, + doc_edus=doc_edus_test) + c_preds.append( + ('ours-tree', load_attelo_ctrees( + EISNER_OUT_TREE_SYN_PRED, EDUS_FILE, + rel_clf_tree, nuc_clf_tree, rnk_clf_tree, + doc_edus=doc_edus_test, + dtree_pred=dtree_pred)) + ) + d_preds.append( + ('ours-tree', dtree_pred) + ) + if author_pred == 'ours-tree-su': + # Eisner, predicted syntax, tree + same-unit + dtree_pred = load_attelo_dtrees( + EISNER_OUT_TREE_SYN_PRED_SU, EDUS_FILE, + rel_clf_tree, nuc_clf_tree, rnk_clf_tree, + doc_edus=doc_edus_test) + c_preds.append( + ('ours-tree-su', load_attelo_ctrees( + EISNER_OUT_TREE_SYN_PRED_SU, EDUS_FILE, + rel_clf_tree, nuc_clf_tree, rnk_clf_tree, + doc_edus=doc_edus_test, + dtree_pred=dtree_pred)) + ) + d_preds.append( + ('ours-tree-su', dtree_pred) + ) + # 2017-05-17 enable "gold" as parser, should give perfect scores + if author_pred == 'gold': + c_preds.append( + ('gold', ctree_true) + ) + d_preds.append( + ('gold', dtree_true) + ) + + if False: # FIXME repair (or forget) these + print('Eisner, predicted syntax + same-unit') + load_deptrees_from_attelo_output( + ctree_true, dtree_true, + EISNER_OUT_SYN_PRED_SU, EDUS_FILE, + rel_clf_chain, nuc_clf_chain, rnk_clf_chain) + print('======================') + + print('Eisner, gold syntax') + load_deptrees_from_attelo_output( + ctree_true, dtree_true, + EISNER_OUT_SYN_GOLD, EDUS_FILE, + rel_clf_chain, nuc_clf_chain, rnk_clf_chain) + print('======================') + + # dependency eval + dep_metrics = ["U"] + if EVAL_NUC_RANK: + dep_metrics += ['O', 'N', 'O+N'] + dep_metrics += ["R"] + if INCLUDE_LS: + dep_metrics += ["tag_R"] + if EVAL_NUC_RANK: + dep_metrics += ["R+N", "F"] # 2017-11-29 disable "R+O" + + # _true + doc_names = sorted(dtree_true.keys()) + labelset_true = set(itertools.chain.from_iterable( + x.labels for x in dtree_true.values())) + labelset_true.add("span") # RST-DT v.1.0 has an error in wsj_1189 7-9 + # 2017-05-17 any author can be used as reference + if author_true != 'each': + parser_true = author_true + print(dep_compact_report(parser_true, d_preds, dep_metrics, + doc_names, labelset_true, + digits=digits, + percent=percent, + out_format=out_fmt)) + else: + print(dep_similarity(d_preds, doc_names, labelset_true, + dep_metric='U', digits=digits, percent=percent, + out_format=out_fmt)) + # raise ValueError("Sim matrix on dependencies not implemented yet") + + # constituency eval + ctree_type = 'SimpleRST' if simple_rsttree else 'RST' + + doc_names = sorted(ctree_true.keys()) + + if False: # back when 'gold' was the only possible ref + ctree_true_list = [ctree_true[doc_name] for doc_name in doc_names] + if simple_rsttree: + ctree_true_list = [SimpleRSTTree.from_rst_tree(x) + for x in ctree_true_list] + # WIP print SimpleRSTTrees + if not os.path.exists('gold'): + os.makedirs('gold') + for doc_name, ct in zip(doc_names, ctree_true_list): + with codecs.open('gold/' + ct.origin.doc, mode='w', + encoding='utf-8') as f: + print(ct, file=f) + + # sort the predictions of each parser, so they match the order of + # documents and reference trees in _true + ctree_preds = [(parser_name, + [ctree_pred[doc_name] for doc_name in doc_names]) + for parser_name, ctree_pred in c_preds] + if simple_rsttree: + ctree_preds = [(parser_name, + [SimpleRSTTree.from_rst_tree(x) + for x in ctree_pred_list]) + for parser_name, ctree_pred_list in ctree_preds] + + # 2017-05-17 allow any parser to be ref + # generate report + if detailed == 0: + # 2017-05-17 WIP similarity matrix: author_true='each': restrict + # to the S metric only, so as to display a sim. matrix + if author_true == 'each': + metric_type = 'S' + print(rst_parseval_similarity(ctree_preds, + ctree_type=ctree_type, + metric_type=metric_type, + digits=digits, + percent=percent, + print_support=False, + per_doc=per_doc, + add_trivial_spans=eval_li_dep, + stringent=STRINGENT, + out_format=out_fmt)) + else: + metric_types = [ + 'S', 'N', 'R', 'F', + 'S+H', 'N+H', 'R+H', 'F+H', + # 'S+K', 'N+K', 'R+K', 'F+K', + # 'S+HH', 'N+HH', 'R+HH', 'F+HH', + # 'S+K+HH', 'N+K+HH', 'R+K+HH', 'F+K+HH', + # 'S+H+K+HH', 'N+H+K+HH', 'R+H+K+HH', 'F+H+K+HH', + ] + # compact report, f1-scores only + print(rst_parseval_compact_report(author_true, ctree_preds, + ctree_type=ctree_type, + metric_types=metric_types, + digits=digits, + percent=percent, + print_support=False, + per_doc=per_doc, + add_trivial_spans=eval_li_dep, + stringent=STRINGENT, + out_format=out_fmt)) + else: + parsers_true = [author_true] if author_true != 'each' else authors_pred + for parser_true in parsers_true: + # standard reports: 1 table per parser, 1 line per metric, + # cols = [p, r, f1, support_true, support_pred] + # FIXME + ctree_true_list = [] + for parser_name, ctree_pred in c_preds: + if parser_name == parser_true: + ctree_true_list = [ctree_pred[doc_name] for doc_name in doc_names] + break + # end FIXME + + for parser_name, ctree_pred_list in ctree_preds: + # WIP print SimpleRSTTrees + if not os.path.exists(parser_name): + os.makedirs(parser_name) + for doc_name, ct in zip(doc_names, ctree_pred_list): + with codecs.open(parser_name + '/' + doc_name, mode='w', + encoding='utf-8') as f: + print(ct, file=f) + + # compute and print PARSEVAL scores + print(parser_name) + # metric_types=None includes the variants with head: + # S+H, N+H, R+H, F+H + print(rst_parseval_report(ctree_true_list, ctree_pred_list, + ctree_type=ctree_type, + metric_types=None, + digits=digits, + percent=percent, + per_doc=per_doc, + add_trivial_spans=eval_li_dep, + stringent=STRINGENT)) + # detailed report on R + if detailed >= 2: + print(rst_parseval_detailed_report( + ctree_true_list, ctree_pred_list, ctree_type=ctree_type, + metric_type='R')) + # end FIXME + + # 2017-04-11 compute agreement between human annotators, on DOUBLE + if 'silver' in authors_pred: + # 'silver' can be meaningfully compared to 'gold' only (too few + # documents otherwise) + if author_true != 'gold': + raise NotImplementedError('Not yet') + + # read the annotation we'll consider as "silver" + reader_dbl = RstReader(DOUBLE_DIR) + corpus_dbl_pred = {k.doc: v for k, v in reader_dbl.slurp().items()} + docs_dbl = sorted(k for k in corpus_dbl_pred.keys()) + # collect the "true" annotation for the docs in double, from train + # and test + # (test has already been read at the beginning of this script) + corpus_test_dbl = {k.doc: v for k, v in corpus_test.items() + if k.doc in docs_dbl} + # read the docs from train that are in double + reader_train = RstReader(CD_TRAIN) + corpus_train = reader_train.slurp() + corpus_train_dbl = {k.doc: v for k, v in corpus_train.items() + if k.doc in docs_dbl} + # assemble the "true" version of the double subset + corpus_dbl_true = dict(corpus_test_dbl.items() + + corpus_train_dbl.items()) + assert (sorted(corpus_dbl_true.keys()) == + sorted(corpus_dbl_pred.keys())) + # extra check? + if False: + for doc_name in docs_dbl: + leaf_spans_true = [x.text_span() for x + in corpus_dbl_true[doc_name].leaves()] + leaf_spans_pred = [x.text_span() for x + in corpus_dbl_pred[doc_name].leaves()] + if (leaf_spans_true != leaf_spans_pred): + print(doc_name, 'EEEE') + print('true - pred', + set(leaf_spans_true) - set(leaf_spans_pred)) + print('pred - true', + set(leaf_spans_pred) - set(leaf_spans_true)) + else: + print(doc_name, 'ok') + # end extra check + + # 48 docs in train, + # 5 docs in test: ['wsj_0627.out', 'wsj_0684.out', 'wsj_1129.out', + # 'wsj_1365.out', 'wsj_1387.out'] + # create parallel lists of ctrees for _true and _pred, mapped to + # coarse rels and binarized + # _pred: + # * ctree + ctree_dbl_pred = [corpus_dbl_pred[doc_name] for doc_name in docs_dbl] + ctree_dbl_pred = [REL_CONV(x) for x in ctree_dbl_pred] + if binarize_true != 'none': # maybe not? + ctree_dbl_pred = [_binarize(x, branching=binarize_true) + for x in ctree_dbl_pred] + # * dtree (as dict from doc_name to dtree !?) + dtree_dbl_pred = {doc_name: RstDepTree.from_rst_tree( + ct, nary_enc=nary_enc_true) + for doc_name, ct in zip(docs_dbl, ctree_dbl_pred)} + # * simple_rsttree (?) + if simple_rsttree: + ctree_dbl_pred = [SimpleRSTTree.from_rst_tree(x) + for x in ctree_dbl_pred] + # _true: + ctree_dbl_true = [corpus_dbl_true[doc_name] for doc_name in docs_dbl] + ctree_dbl_true = [REL_CONV(x) for x in ctree_dbl_true] + if binarize_true != 'none': + ctree_dbl_true = [_binarize(x, branching=binarize_true) + for x in ctree_dbl_true] + # * dtree (as dict from doc_name to dtree !?) + dtree_dbl_true = {doc_name: RstDepTree.from_rst_tree( + ct, nary_enc=nary_enc_true) + for doc_name, ct in zip(docs_dbl, ctree_dbl_true)} + if simple_rsttree: + ctree_dbl_true = [SimpleRSTTree.from_rst_tree(x) + for x in ctree_dbl_true] + # generate report + # * ctree eval + ctree_dbl_preds = [('silver', ctree_dbl_pred), + ('gold', ctree_dbl_true)] + print(rst_parseval_compact_report(author_true, ctree_dbl_preds, + ctree_type=ctree_type, + span_type='chars', + metric_types=['S', 'N', 'R', 'F'], + digits=digits, + percent=percent, + per_doc=per_doc, + add_trivial_spans=eval_li_dep, + stringent=STRINGENT)) + # * dtree eval + if False: + # TODO cope with differences in segmentation + dtree_dbl_preds = [('silver', dtree_dbl_pred), + ('gold', dtree_dbl_true)] + print(dep_compact_report(author_true, dtree_dbl_preds, + dep_metrics, docs_dbl, + labelset_true, + digits=digits, + percent=percent)) + # end 2017-04-11 agreement between human annotators + + +if __name__ == '__main__': + main() diff --git a/evals/surdeanu.py b/evals/surdeanu.py new file mode 100644 index 0000000..7884e34 --- /dev/null +++ b/evals/surdeanu.py @@ -0,0 +1,215 @@ +"""Load RST trees output by Surdeanu et al.'s parser. + +This format differs from the verbose output of the parser: PM added +brackets so they are easier to read. +""" + +from __future__ import absolute_import, print_function +import codecs +import re + +from nltk import Tree + +from educe.annotation import Span +from educe.corpus import FileId +from educe.rst_dt.annotation import EDU, Node, SimpleRSTTree +from educe.rst_dt.deptree import RstDepTree + + +# timestamped line +TS_LINE = r"\d\d:\d\d:\d\d.\d\d\d \[run-main-0\].*" +TS_RE = re.compile(TS_LINE) + + +def tree_to_simple_rsttree(tree, edu_num=1): + """Build a SimpleRSTTree from an NLTK Tree (formatted a la Surdeanu). + + Parameters + ---------- + tree : nltk.Tree + Tree + + edu_num : int, defaults to 1 + Number of the next EDU + + Returns + ------- + sct : SimpleRSTTree + The corresponding SimpleRSTTree. + """ + origin = None + + if tree.label() == 'TEXT': + # EDU (+pre-terminal) + num = edu_num + span = Span(num, num) + # 'TEXT ' + text = '__'.join(tree) + edu = EDU(num, span, text, context=None, origin=origin) + # pre-terminal + edu_span = (num, num) + nuc = "leaf" + rel = "leaf" + node = Node(nuc, edu_span, span, rel, context=None) + return SimpleRSTTree(node, [edu], origin=origin) + + new_kids = [] + for kid in tree: + new_kid = tree_to_simple_rsttree(kid, edu_num=edu_num) + edu_num = new_kid.label().edu_span[1] + 1 + new_kids.append(new_kid) + + # internal node + # (modified) label: 'elaboration:NS' or 'joint' (no explicit nuc: NN) + if tree.label()[-3] == ':': + rel = tree.label()[:-3] + nuc = tree.label()[-2:] + else: + rel = tree.label() + nuc = 'NN' + # map to our coarse rel names + # TODO? + # end map + # same as in braud_coling and braud_eacl + edu_beg = (new_kids[0].num if isinstance(new_kids[0], EDU) + else new_kids[0].label().edu_span[0]) + edu_end = (new_kids[-1].num if isinstance(new_kids[-1], EDU) + else new_kids[-1].label().edu_span[1]) + edu_span = (edu_beg, edu_end) + char_beg = (new_kids[0].num if isinstance(new_kids[0], EDU) + else new_kids[0].label().span.char_start) + char_end = (new_kids[-1].num if isinstance(new_kids[-1], EDU) + else new_kids[-1].label().span.char_end) + span = Span(char_beg, char_end) + new_node = Node(nuc, edu_span, span, rel, context=None) + new_tree = SimpleRSTTree(new_node, new_kids, origin=origin) + return new_tree + + +def _load_surdeanu_ctrees(log_file, rel_conv): + """Do load""" + doc_names = [] + nltk_ctrees = [] + ctree_pred = dict() # result + + ctree_cur = [] # lines for the current c-tree + state_cur = 0 # current state (finite state machine for dummies) + for line in log_file: + # DIRTY replace non-breaking spaces output by CoreNLP, as in + # educe.rst_dt.learning.doc_vectorizer + if isinstance(line, unicode): + line2 = line.replace(u'\xa0', u' ') + line = line2.encode('utf-8') + # end replace + + if state_cur == 0: + line = line.strip() + # skip initial lines until "Documents" + if line == "Documents": + state_cur = 1 + elif state_cur == 1: + line = line.strip() + # read list of document names + if line == "end Documents": + state_cur = 2 + else: + assert line.endswith('.dis') + doc_name = line[:-4] + doc_names.append(doc_name) + elif state_cur == 2: + # skip intermediate lines + if line.strip() == "System tree:": + state_cur = 3 + elif state_cur == 3: + if line.strip() == "System tree:": + if ctree_cur: + # parse the previous predicted c-tree ("System tree") + nltk_ct_pred = Tree.fromstring(''.join(ctree_cur)) + nltk_ctrees.append(nltk_ct_pred) + # reset accumulator + ctree_cur = [] + elif TS_RE.match(line): + # stop reading trees + state_cur = 4 + if ctree_cur: + # parse last predicted tree + nltk_ct_pred = Tree.fromstring(''.join(ctree_cur)) + nltk_ctrees.append(nltk_ct_pred) + ctree_cur = [] # reset (bc who wants side effects?) + else: + # accumulate lines for the next predicted c-tree + # we immediately replace " (LeftToRight)" with ":NS", + # " (RightToLeft)" with ":SN", otherwise it should be ":NN" + line = line.replace(" (LeftToRight)", ":NS").replace(" (RightToLeft)", ":SN").replace("TEXT:", "TEXT ") + ctree_cur.append(line) + elif state_cur == 4: + # just read on + continue + + # we got two predicted ctrees for each doc, with gold then predicted EDUs + # filter to keep only ctrees with gold EDUs, i.e. at even indices + nltk_ctrees = nltk_ctrees[::2] + # for each doc, create an RSTTree from the NLTK tree + for doc_name, nltk_ct_pred in zip(doc_names, nltk_ctrees): + # the c-tree read corresponds to a SimpleRstTree + sct_pred = tree_to_simple_rsttree(nltk_ct_pred) + ct_pred = SimpleRSTTree.to_binary_rst_tree(sct_pred) + ct_pred = rel_conv(ct_pred) + ctree_pred[doc_name] = ct_pred + return ctree_pred + + +def load_surdeanu_ctrees(log_file, rel_conv): + """Load c-trees output by Surdeanu's parser. + + Parameters + ---------- + log_file : str + Path to the log file with the document names followed by the + reference and predicted c-trees. + + rel_conv : RstRelationConverter + Converter to map fine-grained relation labels to classes. + + Returns + ------- + ctree_pred : dict(str, RSTTree) + Predicted c-tree for each doc. + """ + with codecs.open(log_file, mode='rb', encoding='utf-8') as f: + return _load_surdeanu_ctrees(f, rel_conv) + + +def load_surdeanu_dtrees(log_file, rel_conv, nary_enc='chain', + ctree_pred=None): + """Get the dtrees for the ctrees output by Surdeanu's parser. + + Parameters + ---------- + log_file: str + Path to the log file with the output. + rel_conv: TODO + Relation converter, from fine- to coarse-grained labels. + nary_enc: one of {'chain', 'tree'} + Encoding for n-ary nodes. + ctree_pred : dict(str, RSTTree), optional + RST c-trees, indexed by doc_name. If c-trees are provided this + way, `out_dir` is ignored. + + Returns + ------- + dtree_pred: dict(str, RstDepTree) + RST dtree for each document. + """ + dtree_pred = dict() + if ctree_pred is None: + ctree_pred = load_surdeanu_ctrees(log_file, rel_conv) + for doc_name, ct_pred in ctree_pred.items(): + dtree_pred[doc_name] = RstDepTree.from_rst_tree( + ct_pred, nary_enc=nary_enc) + # set reference to the document in the RstDepTree (required by + # dump_disdep_files) + for doc_name, dt_pred in dtree_pred.items(): + dt_pred.origin = FileId(doc_name, None, None, None) + + return dtree_pred diff --git a/evals/train_nuc_classifier.py b/evals/train_nuc_classifier.py new file mode 100644 index 0000000..3126882 --- /dev/null +++ b/evals/train_nuc_classifier.py @@ -0,0 +1,262 @@ +"""This utility script trains a classifier for nuclearity of RST edges. + +Given the path to a nuclearity dataset, it trains a classifier and +evaluates it. +""" + + +from __future__ import absolute_import, print_function + +import argparse +import codecs +from collections import defaultdict +import copy +import itertools +import os +import sys + +from sklearn.datasets import load_svmlight_file, load_svmlight_files +from sklearn.linear_model.logistic import LogisticRegression, LogisticRegressionCV +from sklearn.model_selection import cross_val_score +from sklearn.preprocessing import LabelEncoder + +from educe.rst_dt.annotation import NUC_N, NUC_S + + +# 2017-12-06 non-dummy nuc_clf +# DIRTY load the feature vectors of all candidate edges in the TEST +# set +feat_vecs = dict() +dset_folder = os.path.join( + os.path.expanduser('~'), + 'melodi/rst/irit-rst-dt/TMP/syn_pred_coarse' +) +dset_test = os.path.join(dset_folder, 'TEST.relations.sparse') +# we use the original svmlight files whose label is the relation +# class (which we actually don't need here) +# FIXME read n_features from .vocab +X_test, y_lbl_test = load_svmlight_file(dset_test, n_features=46731, + zero_based=False) +# build mapping from doc_name, src_idx, tgt_idx to line number +# in X_test +pairs = dset_test + '.pairings' +pair_map = defaultdict(lambda: defaultdict(dict)) +with codecs.open(pairs, mode='rb', encoding='utf-8') as f_pairs: + for i, line in enumerate(f_pairs): + src_id, tgt_id = line.strip().split('\t') + src_idx = (0 if src_id == 'ROOT' + else int(src_id.rsplit('_', 1)[1])) + doc_name, tgt_idx = tgt_id.rsplit('_', 1) + tgt_idx = int(tgt_idx) + # print(line) + # print(doc_name, src_idx, tgt_idx) + pair_map[doc_name][src_idx][tgt_idx] = i +# end DIRTY + + +class RightBinaryNuclearityClassifier(object): + """Predict the nuclearity of right-oriented dependencies (binary). + + The nuclearity of ordinary, right-oriented dependencies can be + either `NUC_S` or `NUC_N` (NS or NN relations). + Right-oriented dependencies from the fake root have nuclearity + `NUC_R` by convention ; Left-oriented dependencies have nuclearity + `NUC_S`. + + Parameters + ---------- + bin_clf : sklearn classifier + Binary classifier for right dependencies: NN vs NS. + model_split : str, one of {'none', 'sent', 'sent-para'} + Distinct models for subsets of instances. + """ + + def __init__(self, bin_clf=LogisticRegression(penalty='l1', solver='liblinear', n_jobs=2), model_split='none'): + """Init""" + self.model_split = model_split + if model_split == 'none': + self.bin_clf = bin_clf + elif model_split == 'sent': + self.bin_clf_intra = copy.deepcopy(bin_clf) + self.bin_clf_inter = copy.deepcopy(bin_clf) + else: + raise ValueError("model_split?") + + def fit(self, X, y): + """Fit. + + FIXME X is currently expected to be a (flat) list of candidate + edges instead of a list of RstDepTrees. + """ + if self.model_split == 'none': + self.bin_clf = self.bin_clf.fit(X, y) + if True: # verbose + scores = cross_val_score(self.bin_clf, X, y, cv=10) + print(scores) + print("Accuracy: %0.2f (+/- %0.2f)" % ( + scores.mean(), scores.std() * 2)) + elif self.model_split == 'sent': + assert len(X) == 2 # intra, inter + assert len(y) == 2 # intra, inter + # * intra + self.bin_clf_intra = self.bin_clf_intra.fit(X[0], y[0]) + if True: # verbose + scores = cross_val_score(self.bin_clf_intra, X[0], y[0], cv=10) + print(scores) + print("Accuracy: %0.2f (+/- %0.2f)" % ( + scores.mean(), scores.std() * 2)) + # * inter + self.bin_clf_inter = self.bin_clf_inter.fit(X[1], y[1]) + if True: # verbose + scores = cross_val_score(self.bin_clf_inter, X[1], y[1], cv=10) + print(scores) + print("Accuracy: %0.2f (+/- %0.2f)" % ( + scores.mean(), scores.std() * 2)) + + return self + + def predict(self, X): + """Predict nuclearity of edges in RstDepTrees X from the TEST set. + + Parameters + ---------- + X : list of RstDepTree + D-trees ; the feature vectors of all edges are already + available from the global context. + """ + y = [] + for dtree in X: + doc_name = dtree.origin.doc + yi = [] + for i, head in enumerate(dtree.heads): + if i == 0: + # fake root !? maybe we shouldn't write anything + # here ; + # FIXME check how to be consistent throughout educe and + # eval code + yi.append(NUC_N) + elif i < head: + # left edge: SN + yi.append(NUC_S) + elif head == 0: + # FIXME NUC_R for edges from the root? + yi.append(NUC_N) + else: + # right edge: NN or NS? + line_idx = pair_map[doc_name][head][i] + # X_test[line_idx,:] is a matrix with 1 row + Xi = X_test[line_idx,:] + if self.model_split == 'none': + try: + y_pred = self.bin_clf.predict(Xi) + except ValueError: + print(Xi) + raise + elif self.model_split == 'sent': + # same_sentence_intra_{right,left}: 269, 303 + # our vocab is 1-based but sklearn converts it to + # 0-based ; + # check it's not a left dep + assert Xi[0, 302] == 0 + # + if Xi[0, 268] == 1: + sel_clf = self.bin_clf_intra + else: + sel_clf = self.bin_clf_inter + # + try: + y_pred = sel_clf.predict(Xi) + except ValueError: + print(Xi) + raise + # append prediction + if y_pred == 1: + yi.append(NUC_N) + elif y_pred == 2: + yi.append(NUC_S) + else: + raise ValueError("Weird prediction: {}".format( + y_pred)) + + y.append(yi) + return y + + +if __name__ == "__main__": + model_split = 'sent' # {'none', 'sent'} + # eval on intra- and inter-sent + # * intra + dset_folder_intra = os.path.join( + os.path.expanduser('~'), + 'melodi/rst/irit-rst-dt/TMP/syn_pred_coarse_NUC_intrasent' + ) + dset_train_intra = os.path.join(dset_folder_intra, 'TRAINING.relations.sparse') + dset_test_intra = os.path.join(dset_folder_intra, 'TEST.relations.sparse') + X_train_intra, y_train_intra, X_test_intra, y_test_intra = load_svmlight_files( + (dset_train_intra, dset_test_intra), + n_features=46731, + zero_based=False + ) + # * inter + dset_folder_inter = os.path.join( + os.path.expanduser('~'), + 'melodi/rst/irit-rst-dt/TMP/syn_pred_coarse_NUC_intersent' + ) + dset_train_inter = os.path.join(dset_folder_inter, 'TRAINING.relations.sparse') + dset_test_inter = os.path.join(dset_folder_inter, 'TEST.relations.sparse') + X_train_inter, y_train_inter, X_test_inter, y_test_inter = load_svmlight_files( + (dset_train_inter, dset_test_inter), + n_features=46731, + zero_based=False + ) + # + if model_split == 'none': + # import the nuclearity TRAIN and TEST sets + dset_folder = os.path.join( + os.path.expanduser('~'), + 'melodi/rst/irit-rst-dt/TMP/syn_pred_coarse_NUC' + ) + dset_train = os.path.join(dset_folder, 'TRAINING.relations.sparse') + dset_test = os.path.join(dset_folder, 'TEST.relations.sparse') + + X_train, y_train, X_test, y_test = load_svmlight_files( + (dset_train, dset_test), + n_features=46731, + zero_based=False + ) + nuc_clf = LogisticRegressionCV(penalty='l1', solver='liblinear', + n_jobs=3) + # train nuclearity classifier, cross-validate performance on train + scores = cross_val_score(nuc_clf, X_train, y_train, cv=10) + print(scores) + print("Accuracy: %0.2f (+/- %0.2f)" % (scores.mean(), scores.std() * 2)) + # fit a + nuc_clf = nuc_clf.fit(X_train, y_train) + print(nuc_clf.score(X_test, y_test)) + print('separate eval on intra then inter') + print(nuc_clf.score(X_test_intra, y_test_intra)) + print(nuc_clf.score(X_test_inter, y_test_inter)) + elif model_split == 'sent': + # fit distinct classifiers for intra- and inter-sentential + # * intra: train nuclearity classifier, cross-validate performance on train + nuc_clf_intra = LogisticRegressionCV(penalty='l1', solver='liblinear', + n_jobs=3) + scores_intra = cross_val_score(nuc_clf_intra, X_train_intra, y_train_intra, + cv=10) + print(scores_intra) + print("Accuracy: %0.2f (+/- %0.2f)" % ( + scores_intra.mean(), scores_intra.std() * 2)) + # + nuc_clf_intra = nuc_clf_intra.fit(X_train_intra, y_train_intra) + print(nuc_clf_intra.score(X_test_intra, y_test_intra)) + # * inter: train nuclearity classifier, cross-validate performance on train + nuc_clf_inter = LogisticRegressionCV(penalty='l1', solver='liblinear', + n_jobs=3) + scores_inter = cross_val_score(nuc_clf_inter, X_train_inter, y_train_inter, + cv=10) + print(scores_inter) + print("Accuracy: %0.2f (+/- %0.2f)" % ( + scores_inter.mean(), scores_inter.std() * 2)) + # + nuc_clf_inter = nuc_clf_inter.fit(X_train_inter, y_train_inter) + print(nuc_clf_inter.score(X_test_inter, y_test_inter)) diff --git a/evals/train_rel_relabeller.py b/evals/train_rel_relabeller.py new file mode 100644 index 0000000..4ccf661 --- /dev/null +++ b/evals/train_rel_relabeller.py @@ -0,0 +1,201 @@ +"""This utility script trains a (re)labeller for RST edges. + +Given the path to a relation labelling dataset, it trains a classifier +and evaluates it. +""" + +from __future__ import absolute_import, print_function + +import argparse +import codecs +from collections import defaultdict +import copy +import os + +from sklearn.datasets import load_svmlight_file, load_svmlight_files +from sklearn.linear_model.logistic import LogisticRegression, LogisticRegressionCV +from sklearn.model_selection import cross_val_score + +from educe.rst_dt.deptree import _ROOT_HEAD, _ROOT_LABEL + + +# build mapping from int to label (reverse label encoding) +dset_rel_folder = os.path.join( + os.path.expanduser('~'), + 'melodi/rst/irit-rst-dt/TMP/syn_pred_coarse_REL' +) +dset_rel_train = os.path.join(dset_rel_folder, 'TRAINING.relations.sparse') +dset_rel_test = os.path.join(dset_rel_folder, 'TEST.relations.sparse') + +with codecs.open(dset_rel_train, mode='rb', encoding='utf-8') as f_train: + header = f_train.readline() + header_prefix = '# labels: ' + assert header.startswith(header_prefix) + # DEBUG? explicit cast from unicode to str + labels = [str(lbl) for lbl in header[len(header_prefix):].split()] + int2lbl = dict(enumerate(labels, start=1)) + lbl2int = {lbl: i for i, lbl in int2lbl.items()} + # unrelated = lbl2int["UNRELATED"] + # root = lbl2int["ROOT"] + +# 2017-12-14 relation (re)labeller +# DIRTY load the feature vector for all *candidate* edges in the TEST +# set (for predict()) +feat_vecs = dict() +dset_folder = os.path.join( + os.path.expanduser('~'), + 'melodi/rst/irit-rst-dt/TMP/syn_pred_coarse' +) +dset_test = os.path.join(dset_folder, 'TEST.relations.sparse') +# we use the original svmlight files whose label is the relation +# class (which we actually don't need here) +# FIXME read n_features from .vocab +X_test, y_lbl_test = load_svmlight_file(dset_test, n_features=46731, + zero_based=False) +# build mapping from doc_name, src_idx, tgt_idx to line number +# in X_test +pairs = dset_test + '.pairings' +pair_map = defaultdict(lambda: defaultdict(dict)) +with codecs.open(pairs, mode='rb', encoding='utf-8') as f_pairs: + for i, line in enumerate(f_pairs): + src_id, tgt_id = line.strip().split('\t') + src_idx = (0 if src_id == 'ROOT' + else int(src_id.rsplit('_', 1)[1])) + doc_name, tgt_idx = tgt_id.rsplit('_', 1) + tgt_idx = int(tgt_idx) + # print(line) + # print(doc_name, src_idx, tgt_idx) + pair_map[doc_name][src_idx][tgt_idx] = i +# end DIRTY + + +if False: + # load the relation TRAIN and TEST sets + X_rel_train, y_rel_train, X_rel_test, y_rel_test = load_svmlight_files( + (dset_rel_train, dset_rel_test), + zero_based=False + ) + rel_clf = LogisticRegressionCV(penalty='l1', solver='liblinear', + n_jobs=3) + # train relation classifier, cross-validate performance on train + scores = cross_val_score(rel_clf, X_rel_train, y_rel_train, cv=10) + print(scores) + print("Accuracy: %0.2f (+/- %0.2f)" % (scores.mean(), scores.std() * 2)) + # fit a + rel_clf = rel_clf.fit(X_rel_train, y_rel_train) + print(rel_clf.score(X_rel_test, y_rel_test)) + + +class RelationRelabeller(object): + """Predict the coarse-grained RST relation of dependencies. + + Dependencies headed by the fake root node are labelled "ROOT" by + convention. + + Parameters + ---------- + mul_clf : sklearn classifier + Multi-class classifier for RST (coarse-grained) relations. + """ + + def __init__(self, mul_clf=LogisticRegression(penalty='l1', solver='liblinear', n_jobs=3), model_split='none'): + """Init""" + self.model_split = model_split + if model_split == 'none': + self.mul_clf = mul_clf + elif model_split == 'sent': + self.mul_clf_intra = copy.deepcopy(mul_clf) + self.mul_clf_inter = copy.deepcopy(mul_clf) + else: + raise ValueError("model_split?") + + def fit(self, X, y): + """Fit. + + FIXME X is currently expected to be a (flat) list of candidate + edges instead of a list of RstDepTrees. + """ + if self.model_split == 'none': + self.mul_clf = self.mul_clf.fit(X, y) + if True: # verbose + scores = cross_val_score(self.mul_clf, X, y, cv=10) + print(scores) + print("Accuracy: %0.2f (+/- %0.2f)" % ( + scores.mean(), scores.std() * 2)) + elif self.model_split == 'sent': + assert len(X) == 2 # intra, inter + assert len(y) == 2 # intra, inter + # * intra + self.mul_clf_intra = self.mul_clf_intra.fit(X[0], y[0]) + if True: # verbose + scores = cross_val_score(self.mul_clf_intra, X[0], y[0], cv=10) + print(scores) + print("Accuracy: %0.2f (+/- %0.2f)" % ( + scores.mean(), scores.std() * 2)) + # * inter + self.mul_clf_inter = self.mul_clf_inter.fit(X[1], y[1]) + if True: # verbose + scores = cross_val_score(self.mul_clf_inter, X[1], y[1], cv=10) + print(scores) + print("Accuracy: %0.2f (+/- %0.2f)" % ( + scores.mean(), scores.std() * 2)) + + return self + + def predict(self, X): + """Predict relation of edges in RstDepTrees X from the TEST set. + """ + y = [] + for dtree in X: + doc_name = dtree.origin.doc + yi = [] + for i, (head, rel) in enumerate(zip(dtree.heads, dtree.labels)): + if i == 0: + # fake root !? maybe we shouldn't write anything + # here ; + # FIXME check how to be consistent throughout educe and + # eval code + # yi.append(_ROOT_LABEL) + yi.append(None) + elif head == 0: + # TODO check the expected value (consistency) + yi.append(_ROOT_LABEL) + else: + # regular edge + line_idx = pair_map[doc_name][head][i] + # X_test[line_idx,:] is a matrix with 1 row + Xi = X_test[line_idx,:] + if self.model_split == 'none': + try: + y_pred = self.mul_clf.predict(Xi) + except ValueError: + print(Xi) + raise + elif self.model_split == 'sent': + # same_sentence_intra_{right,left}: 269, 303 + # our vocab is 1-based but sklearn converts it to + # 0-based ; + # same_para_* : 103, 158, 234, 314 + if ((Xi[0, 268] == 1 or Xi[0, 302] == 1) and + (Xi[0, 102] == 1 or Xi[0, 157] == 1 or + Xi[0, 233] == 1 or Xi[0, 313] == 1)): + sel_clf = self.mul_clf_intra + else: + sel_clf = self.mul_clf_inter + # + try: + y_pred = sel_clf.predict(Xi) + except ValueError: + print(Xi) + raise + # append prediction + try: + yi.append(int2lbl[int(y_pred[0])]) + if False and rel != int2lbl[int(y_pred[0])]: + print(doc_name, head, i, + rel, int2lbl[int(y_pred[0])]) # DEBUG + except KeyError: + raise ValueError("Weird prediction: {}".format( + y_pred)) + y.append(yi) + return y diff --git a/evals/utils_wip.py b/evals/utils_wip.py new file mode 100644 index 0000000..bd1d1d0 --- /dev/null +++ b/evals/utils_wip.py @@ -0,0 +1,248 @@ +"""Various utility functions that are WIP. + +These functions are expected to move to educe or attelo when they +are mature. +""" + +from __future__ import print_function + +import os +import sys + +from educe.rst_dt.annotation import RSTTree +from educe.rst_dt.corpus import Reader as RstReader +from educe.rst_dt.dep2con import deptree_to_simple_rst_tree +from educe.rst_dt.deptree import RstDepTree, RstDtException +# +from evals.ours import load_attelo_output_file + + +# RST corpus +CORPUS_DIR = os.path.abspath(os.path.join( + os.path.dirname(os.path.realpath(__file__)), + '..', 'corpus', + 'RSTtrees-WSJ-main-1.0/')) +CD_TRAIN = os.path.join(CORPUS_DIR, 'TRAINING') +CD_TEST = os.path.join(CORPUS_DIR, 'TEST') + +# move to educe.rst_dt.datasets.rst_dis_format +STR_ROOT = '{nuc} (span {edu_span})' +STR_NODE = '{nuc} (span {edu_span}) (rel2par {rel})' +STR_LEAF = '{nuc} (leaf {edu_num}) (rel2par {rel}) (text _!{edu_txt}_!)' + + +def _str_node(tree): + """String for the top node of an RSTTree + + Parameters + ---------- + tree: educe.rst_dt.annotation.RSTTree + The tree whose top node we want to print + """ + node = tree.label() + # get fields + nuc = node.nuclearity + edu_span = node.edu_span + rel = node.rel + # leaf (in reality, we are at the pre-terminal) + if len(tree) == 1: + # get text from the real leaf (EDU) + txt = tree[0].text() + node_str = STR_LEAF.format(nuc=nuc, edu_num=edu_span[0], + rel=rel, edu_txt=txt) + # internal node + else: + edu_span_str = '{} {}'.format(str(edu_span[0]), str(edu_span[1])) + node_str = STR_NODE.format(nuc=nuc, edu_span=edu_span_str, + rel=rel) + + return node_str + + +def tree_str_gen(tree): + """Return a generator of strings, one per tree node""" + # init tree stack with the whole tree, nesting level 0 + tree_stack = [(tree, 0)] + + while tree_stack: + tree, lvl = tree_stack.pop() + yield '{lw}{node_str}'.format(lw=' ' * lvl, + node_str=_str_node(tree)) + tree_stack.extend(reversed([(subtree, lvl + 1) for subtree in tree + if isinstance(subtree, RSTTree)])) + # RESUME HERE: add opening (easy) and closing (trickier) parentheses + # TODO do not print relation (None) for ROOT + + +def _dump_rst_dis_file(out_file, ct_pred): + """Actually do dump. + + Parameters + ---------- + out_file: File + Output file + + ct_pred: RSTTree + Binary RST tree + """ + res_str = '\n'.join(tree_str_gen(ct_pred)) # or str(ct_pred) ? + out_file.write(res_str) + + +def dump_rst_dis_file(out_file, ctree): + """Dump a binary RST tree to a file. + + Parameters + ---------- + out_file: string + Path to the output file + + ctree: RSTTree + Binary RST tree + """ + with open(out_file, 'w') as f: + _dump_rst_dis_file(f, ctree) +# end educe.rst_dt.datasets.rst_dis_format + + +# move to educe.rst_dt.datasets.dep_dis_format ? +def dump_dep_dis_file(out_file, dtree): + """Dump a (RST) dependency tree to a file. + + Parameters + ---------- + out_file: string + Path to the output file + + dtree: RstDepTree + RST dependency tree + """ + with open(out_file, 'w') as f: + res = '\n'.join('{}\t{}'.format(hd, lbl) + for hd, lbl in zip(dtree.heads, dtree.labels)) + f.write(res) +# end attelo.datasets.dep_dis_format + + +# move to educe.rst_dt.attelo_out_format +# +# this function is only called by `convert_attelo_output_file_to_dis_files` +# +# FIXME: find ways to read the right (not necessarily TEST) section +# and only the required documents +def load_trees_from_attelo_output_file(att_output_file): + """Load predicted RST trees from attelo's output file. + + Parameters + ---------- + att_output_file: string + Path to the file that contains attelo's output + + Returns + ------- + ctrees_pred: dict(string, SimpleRSTTree) + Predicted SimpleRSTTree for each document, indexed by its name + """ + # get predicted tree for each doc + # these currently come in the form of edges on attelo EDUs + edges_pred = load_attelo_output_file(att_output_file) + + # get educe EDUs + edus = dict() + # FIXME: parameterize this, cf. function-wide FIXME above + rst_reader = RstReader(CD_TEST) + rst_corpus = rst_reader.slurp() + for doc_id, rtree_true in sorted(rst_corpus.items()): + doc_name = doc_id.doc + edus[doc_name] = rtree_true.leaves() + + # re-build predicted trees from predicted edges and educe EDUs + dtree_pred = dict() # predicted dtrees + ctree_pred = dict() # predicted ctrees + skipped_docs = set() # docs skipped because non-projective structures + for doc_name, es_pred in sorted(edges_pred.items()): + # map from EDU id to EDU num + # EDU id should be common to educe and attelo + id2num = {edu.identifier(): edu.num for edu in edus[doc_name]} + # create pred dtree + dt_pred = RstDepTree(edus[doc_name]) + for src_id, tgt_id, lbl in es_pred: + if src_id == 'ROOT': + if lbl == 'ROOT': + dt_pred.set_root(id2num[tgt_id]) + else: + raise ValueError('Weird root label: {}'.format(lbl)) + else: + dt_pred.add_dependency(id2num[src_id], id2num[tgt_id], lbl) + dtree_pred[doc_name] = dt_pred + # create pred ctree + try: + ctree_pred[doc_name] = deptree_to_simple_rst_tree(dt_pred) + except RstDtException: + skipped_docs.add(doc_name) + if False: + print('\n'.join('{}: {}'.format(edu.text_span(), edu) + for edu in edus[doc_name])) + # raise + if skipped_docs: + print('Skipped {} docs over {}'.format(len(skipped_docs), + len(edges_pred))) + + return ctree_pred +# end educe.rst_dt.attelo_out_format + + +# move to educe.datasets.rst_dis_format +def convert_attelo_output_file_to_dis_files(output_dir, att_output_file): + """Convert attelo's output file to a set of dis files in output_dir. + + Parameters + ---------- + output_dir: string + Path of the directory for the dis files + output_file: string + Path to the file that contains attelo's output + + Returns + ------- + ctrees_pred: dict(string, SimpleRSTTree) + Predicted SimpleRSTTree for each document, indexed by its name + """ + if not os.path.exists(output_dir): + raise ValueError('Absent path: {}'.format(output_dir)) + + ctree_pred = load_trees_from_attelo_output_file(att_output_file) + # output each SimpleRSTTree to a dis file + for doc_name, ct_pred in ctree_pred.items(): + out_fname = os.path.join(output_dir, doc_name + '.dis') + dump_rst_dis_file(out_fname, ct_pred) + # DEBUG + sys.exit() +# end educe.datasets.rst_dis_format + + +# ?? +def load_gold(): + """Load gold structures from RST-WSJ/TEST. + + Returns + ------- + data: dictionary that should be akin to a sklearn Bunch, + with interesting keys 'filenames', 'doc_names', 'rst_ctrees', + 'rst_dtrees'. + """ + # TODO make this the only place where the gold is loaded + # shared between evals of both CODRA and attelo's outputs + filenames = [] # TODO + # load doc names and reference trees + rst_reader = RstReader(CD_TEST) + rst_corpus = rst_reader.slurp() + doc_names = [] + rst_ctrees = [] + for doc_id, rst_ctree in sorted(rst_corpus.items(), + key=lambda kv: kv[0].doc): + doc_names.append(doc_id.doc) + rst_ctrees.append(rst_ctree) + # RESUME HERE (or not) + raise NotImplementedError +# end ?? diff --git a/irit_rst_dt/cmd/clean.py b/irit_rst_dt/cmd/clean.py index ad55823..2a9a019 100644 --- a/irit_rst_dt/cmd/clean.py +++ b/irit_rst_dt/cmd/clean.py @@ -34,10 +34,10 @@ def main(_): You shouldn't need to call this yourself if you're using `config_argparser` """ - for data_dir in sorted(subdirs(LOCAL_TMP)): - if fp.basename(data_dir) == "latest": + for base_dir in sorted(subdirs(LOCAL_TMP)): + if fp.basename(base_dir) == "latest": continue - for subdir in subdirs(data_dir): + for subdir in subdirs(base_dir): bname = fp.basename(subdir) if bname in ["eval-current", "eval-previous", "scratch-current", "scratch-previous"]: diff --git a/irit_rst_dt/cmd/gather.py b/irit_rst_dt/cmd/gather.py index f7fcba6..df37074 100644 --- a/irit_rst_dt/cmd/gather.py +++ b/irit_rst_dt/cmd/gather.py @@ -6,17 +6,17 @@ """ from __future__ import print_function -from os import path as fp +import itertools import os from attelo.harness.util import call, force_symlink +from attelo.learning.oracle import AttachOracle +from attelo.parser.intra import IntraInterParser +from attelo.parser.same_unit import SameUnitClassifierWrapper -from ..local import (TEST_CORPUS, - TRAINING_CORPUS, - PTB_DIR, - FEATURE_SET, - CORENLP_OUT_DIR, - LECSIE_DATA_DIR) +from ..local import (FEATURE_SET, LABEL_SET, TEST_CORPUS, TRAINING_CORPUS, + SAME_UNIT, PTB_DIR, CORENLP_OUT_DIR, LECSIE_DATA_DIR, + NARY_ENC, EVALUATIONS) from ..util import (current_tmp, latest_tmp) NAME = 'gather' @@ -40,16 +40,19 @@ def config_argparser(psr): psr.add_argument('--skip-training', action='store_true', help='only gather test data') - psr.add_argument('--coarse', - action='store_true', - help='use coarse-grained labels') psr.add_argument('--fix_pseudo_rels', - action='store_true', - help='fix pseudo-relation labels') + action='store_true', + help='fix pseudo-relation labels') + # WIP frag pairs + psr.add_argument('--resume-frag-pairs', + action='store_true', + help='resume extraction at frag-pairs') + # end WIP frag pairs psr.set_defaults(func=main) -def extract_features(corpus, output_dir, coarse, fix_pseudo_rels, +def extract_features(corpus, output_dir, fix_pseudo_rels, instances, + frag_edus=None, vocab_path=None, label_path=None): """Extract instances from a corpus, store them in files. @@ -64,10 +67,10 @@ def extract_features(corpus, output_dir, coarse, fix_pseudo_rels, Path to the corpus. output_dir: filepath Path to the output folder. - coarse: boolean, False by default - Use coarse-grained relation labels. fix_pseudo_rels: boolean, False by default Rewrite pseudo-relations to improve consistency (WIP). + instances: one of {'same-unit', 'edu-pairs'} + Selection of instances to extract. vocab_path: filepath Path to a fixed vocabulary mapping, for feature extraction (needed if extracting test data: the same vocabulary should be @@ -83,6 +86,8 @@ def extract_features(corpus, output_dir, coarse, fix_pseudo_rels, PTB_DIR, # TODO make this optional and exclusive from CoreNLP output_dir, '--feature_set', FEATURE_SET, + '--nary_enc', NARY_ENC, # 2016-09-12 + '--instances', instances, ] # NEW 2016-05-19 rewrite pseudo-relations if fix_pseudo_rels: @@ -91,7 +96,7 @@ def extract_features(corpus, output_dir, coarse, fix_pseudo_rels, ]) # NEW 2016-05-03 use coarse- or fine-grained relation labels # NB "coarse" was the previous default - if coarse: + if LABEL_SET == 'coarse': cmd.extend([ '--coarse' ]) @@ -103,6 +108,8 @@ def extract_features(corpus, output_dir, coarse, fix_pseudo_rels, cmd.extend([ '--lecsie_data_dir', LECSIE_DATA_DIR, ]) + if frag_edus is not None: + cmd.extend(['--frag-edus', frag_edus]) if vocab_path is not None: cmd.extend(['--vocabulary', vocab_path]) if label_path is not None: @@ -117,22 +124,99 @@ def main(args): You shouldn't need to call this yourself if you're using `config_argparser` """ - if args.skip_training: + if args.skip_training or args.resume_frag_pairs: tdir = latest_tmp() else: tdir = current_tmp() - extract_features(TRAINING_CORPUS, tdir, args.coarse, - args.fix_pseudo_rels) - if TEST_CORPUS is not None: - train_path = fp.join(tdir, fp.basename(TRAINING_CORPUS)) - label_path = train_path + '.relations.sparse' - vocab_path = label_path + '.vocab' - extract_features(TEST_CORPUS, tdir, args.coarse, - args.fix_pseudo_rels, + + fix_pseudo_rels = args.fix_pseudo_rels + + # 2016-09-01 put data files in {tdir}/data + tdir_data = os.path.join(tdir, 'data') + if not os.path.exists(tdir_data): + os.makedirs(tdir_data) + # same-unit + all_parsers = [] + for econf in EVALUATIONS: + parser = econf.parser[1] + if isinstance(parser, IntraInterParser): + all_parsers.extend(x[1] for x in itertools.chain( + parser._parsers.intra.steps, parser._parsers.inter.steps)) + else: + all_parsers.extend(x[1] for x in parser.steps) + same_unit_parsers = [x for x in all_parsers + if isinstance(x, SameUnitClassifierWrapper)] + same_unit_clfs = [x._learner_su for x in same_unit_parsers] + if same_unit_parsers and not args.resume_frag_pairs: + instances = 'same-unit' + su_prefix_train = '{}.relations.{}'.format( + os.path.basename(TRAINING_CORPUS), instances) + su_train_path = os.path.join(tdir_data, su_prefix_train) + su_label_path = su_train_path + '.labels' + su_vocab_path = su_train_path + '.sparse.vocab' + if TEST_CORPUS is not None: + su_prefix_test = '{}.{}'.format( + os.path.basename(TEST_CORPUS), instances) + su_test_path = os.path.join(tdir_data, su_prefix_test) + + if not args.skip_training: + # * train + extract_features(TRAINING_CORPUS, tdir_data, fix_pseudo_rels, + instances) + if TEST_CORPUS is not None: + # * test + extract_features(TEST_CORPUS, tdir_data, fix_pseudo_rels, + instances, + vocab_path=su_vocab_path, + label_path=su_label_path) + + # all pairs + instances = 'edu-pairs' + if not args.skip_training and not args.resume_frag_pairs: + extract_features(TRAINING_CORPUS, tdir_data, fix_pseudo_rels, + instances) + # path to the vocab and labelset gathered from the training set, + # we'll use these paths for the test set and for the frag-pairs + prefix_train = '{}.relations.{}'.format( + os.path.basename(TRAINING_CORPUS), instances) + train_path = os.path.join(tdir_data, prefix_train) + label_path = train_path + '.labels' + vocab_path = train_path + '.sparse.vocab' + if TEST_CORPUS is not None and not args.resume_frag_pairs: + extract_features(TEST_CORPUS, tdir_data, fix_pseudo_rels, + instances, vocab_path=vocab_path, label_path=label_path) - with open(os.path.join(tdir, "versions-gather.txt"), "w") as stream: + + # WIP 2017-02-03 disable frag-pairs + if False: + # frag pairs: supplementary pairs from/to each fragmented EDU to + # the other fragmented EDUs and the EDUs that don't belong to any + # fragmented EDU + instances = 'frag-pairs' + same_unit_types = set(('true' if isinstance(x, AttachOracle) + else 'pred') + for clf in same_unit_clfs) + for same_unit_type in sorted(same_unit_types): + # we use the vocabulary and labelset from "edu-pairs" ; + # this is the simplest solution currently and it seems + # correct, but maybe we could extend "edu-pairs" with these + # pairs when we learn the vocabulary? + if not args.skip_training: + extract_features(TRAINING_CORPUS, tdir_data, fix_pseudo_rels, + instances, frag_edus=same_unit_type, + vocab_path=vocab_path, + label_path=label_path) + if TEST_CORPUS is not None: + extract_features(TEST_CORPUS, tdir_data, fix_pseudo_rels, + instances, frag_edus=same_unit_type, + vocab_path=vocab_path, + label_path=label_path) + # end frag pairs + + with open(os.path.join(tdir_data, "versions-gather.txt"), "w") as stream: call(["pip", "freeze"], stdout=stream) - if not args.skip_training: + + if not (args.skip_training or args.resume_frag_pairs): latest_dir = latest_tmp() - force_symlink(fp.basename(tdir), latest_dir) + force_symlink(os.path.basename(tdir), latest_dir) diff --git a/irit_rst_dt/config/common.py b/irit_rst_dt/config/common.py index 54a6f2e..1ff9e9b 100644 --- a/irit_rst_dt/config/common.py +++ b/irit_rst_dt/config/common.py @@ -1,7 +1,9 @@ """Commonly used configuration options""" from collections import namedtuple +import copy import six + # from attelo.decoding.astar import (AstarArgs, # AstarDecoder, # Heuristic, @@ -14,6 +16,8 @@ from attelo.learning.oracle import (AttachOracle, LabelOracle) from attelo.parser.full import (JointPipeline, PostlabelPipeline) +from attelo.parser.same_unit import (JointSameUnitPipeline, + SameUnitJointPipeline) def combined_key(*variants): @@ -115,6 +119,44 @@ def mk_joint(klearner, kdecoder): parser=Keyed(parser_key, parser)) +def mk_joint_su(klearner, kdecoder): + "return a joint decoding parser config with same-unit" + settings = _core_settings('AD.L-jnt_su', klearner) + parser_key = combined_key(settings, kdecoder) + key = combined_key(klearner, parser_key) + # su: use same kind of learner as "attach" + parser = JointSameUnitPipeline( + learner_attach=klearner.attach.payload, + learner_label=klearner.label.payload, + # FIXME this copy does not really make sense here, but at least + # its type is correct + learner_su=copy.deepcopy(klearner.attach.payload), + decoder=kdecoder.payload) + return EvaluationConfig(key=key, + settings=settings, + learner=klearner, + parser=Keyed(parser_key, parser)) + + +def mk_su_joint(klearner, kdecoder): + "return a parser config with same-unit then joint decoding" + settings = _core_settings('su.AD.L-jnt', klearner) + parser_key = combined_key(settings, kdecoder) + key = combined_key(klearner, parser_key) + # su: use same kind of learner as "attach" + parser = JointSameUnitPipeline( + # FIXME this copy does not really make sense here, but at least + # its type is correct + learner_su=copy.deepcopy(klearner.attach.payload), + learner_attach=klearner.attach.payload, + learner_label=klearner.label.payload, + decoder=kdecoder.payload) + return EvaluationConfig(key=key, + settings=settings, + learner=klearner, + parser=Keyed(parser_key, parser)) + + def mk_post(klearner, kdecoder): "return a post label parser" settings = _core_settings('AD.L-pst', klearner) diff --git a/irit_rst_dt/config/intra.py b/irit_rst_dt/config/intra.py index b130ab5..7806d29 100644 --- a/irit_rst_dt/config/intra.py +++ b/irit_rst_dt/config/intra.py @@ -11,17 +11,17 @@ def combine_intra(econfs, kconf, primary='intra', verbose=False): Parameters ---------- econfs: IntraInterPair(EvaluationConfig) - + Evaluation configs for the intra and inter parsers. kconf: Keyed(parser constructor) - - primary: ['intra', 'inter'] - Treat the intra/inter config as the primary one for the key + Key for the whole intra/inter parser. + primary: one of {'intra', 'inter'} + Treat the intra or inter config as the primary one for the key. verbose: boolean, optional Verbosity of the intra/inter parser Returns ------- - econf : EvaluationConfig + econf: EvaluationConfig Evaluation configuration for the IntraInterParser. """ if primary == 'intra': @@ -31,9 +31,9 @@ def combine_intra(econfs, kconf, primary='intra', verbose=False): else: raise ValueError("'primary' should be one of intra/inter: " + primary) - parsers = econfs.fmap(lambda e: e.parser.payload) - subsettings = econfs.fmap(lambda e: e.settings) - learners = econfs.fmap(lambda e: e.learner) + parsers = econfs.fmap(lambda e: e.parser.payload) # IntraInterPair + subsettings = econfs.fmap(lambda e: e.settings) # IntraInterPair + learners = econfs.fmap(lambda e: e.learner) # IntraInterPair settings = Settings(key=combined_key(kconf, econf.settings), intra=True, oracle=econf.settings.oracle, diff --git a/irit_rst_dt/harness.py b/irit_rst_dt/harness.py index 469ce2b..726ef05 100644 --- a/irit_rst_dt/harness.py +++ b/irit_rst_dt/harness.py @@ -2,6 +2,7 @@ Paths to files used or generated by the test harness ''' from collections import Counter +from glob import glob from os import path as fp import sys @@ -42,13 +43,13 @@ def __init__(self): def run(self, runcfg): """Run the evaluation """ - data_dir = latest_tmp() - if not fp.exists(data_dir): + base_dir = latest_tmp() + if not fp.exists(base_dir): exit_ungathered() - eval_dir, scratch_dir = prepare_dirs(runcfg, data_dir) + eval_dir, scratch_dir = prepare_dirs(runcfg, base_dir) self.load(runcfg, eval_dir, scratch_dir) evidence_of_gathered = self.mpack_paths(False)['edu_input'] - if not fp.exists(evidence_of_gathered): + if not glob(evidence_of_gathered): exit_ungathered() evaluate_corpus(self) @@ -106,7 +107,6 @@ def create_folds(self, mpack): # ------------------------------------------------------ # paths # ------------------------------------------------------ - def mpack_paths(self, test_data, stripped=False): """Return a dict of paths needed to read a datapack. @@ -126,10 +126,13 @@ def mpack_paths(self, test_data, stripped=False): Useful keys are 'edu_input', 'pairings', 'features', 'vocab', 'corpus' (WIP, used to access gold structures). """ - ext = 'relations.sparse' + base = 'relations.edu-pairs' + ext = base + '.sparse' # path to data file in the evaluation dir dset = self.testset if test_data else self.dataset - core_path = fp.join(self.eval_dir, "%s.%s" % (dset, ext)) + vocab_path = fp.join(self.eval_dir, "%s.%s.vocab" % (dset, ext)) + labels_path = fp.join(self.eval_dir, "%s.%s.labels" % (dset, base)) + core_path = fp.join(self.eval_dir, dset, "*.%s" % ext) # WIP gold RST trees corpus_path = fp.abspath(TEST_CORPUS if test_data else TRAINING_CORPUS) @@ -203,7 +206,8 @@ def _eval_model_path(subconf, mtype): else: return { 'attach': _eval_model_path(rconf, "attach"), - 'label': _eval_model_path(rconf, "relate") + 'label': _eval_model_path(rconf, "relate"), + 'su': _eval_model_path(rconf, "su"), } # ------------------------------------------------------ diff --git a/irit_rst_dt/local.py b/irit_rst_dt/local.py index 4fad6a4..ab5c087 100644 --- a/irit_rst_dt/local.py +++ b/irit_rst_dt/local.py @@ -11,7 +11,13 @@ from os import path as fp import itertools as itr -from attelo.harness.config import (LearnerConfig, +from sklearn.linear_model import (LogisticRegression) +from sklearn.tree import DecisionTreeClassifier +from sklearn.ensemble import RandomForestClassifier + +# attelo +from attelo.harness.config import (EvaluationConfig, + LearnerConfig, Keyed) # from attelo.decoding.astar import (AstarArgs, # AstarDecoder, @@ -21,17 +27,16 @@ from attelo.decoding.mst import (MstDecoder, MstRootStrategy) from attelo.learning.local import (SklearnAttachClassifier, SklearnLabelClassifier) +from attelo.learning.oracle import AttachOracle from attelo.parser.intra import (IntraInterPair, HeadToHeadParser, FrontierToHeadParser, # SentOnlyParser, SoftParser) +from attelo.parser.same_unit import (JointSameUnitPipeline, + SameUnitJointPipeline) -from sklearn.linear_model import (LogisticRegression) -from sklearn.tree import DecisionTreeClassifier -from sklearn.ensemble import RandomForestClassifier - - +# this harness from .config.intra import (combine_intra) from .config.perceptron import (attach_learner_dp_pa, attach_learner_dp_perc, @@ -49,7 +54,11 @@ decoder_last, decoder_local, mk_joint, - mk_post) + mk_joint_su, + mk_su_joint, + mk_post, + JointPipeline, + Settings) # PATHS @@ -64,8 +73,8 @@ """Results over time we are making a point of saving""" # TRAINING_CORPUS = 'tiny' -# TRAINING_CORPUS = 'corpus/RSTtrees-WSJ-main-1.0/TRAINING' -TRAINING_CORPUS = 'corpus/RSTtrees-WSJ-double-1.0' +TRAINING_CORPUS = 'corpus/RSTtrees-WSJ-main-1.0/TRAINING' +# TRAINING_CORPUS = 'corpus/RSTtrees-WSJ-double-1.0' """Corpora for use in building/training models and running our incremental experiments. Later on we should consider using the held-out test data for something, but let's make a point of @@ -90,10 +99,12 @@ validation on the training data) """ -TEST_EVALUATION_KEY = None +# TEST_EVALUATION_KEY = None # TEST_EVALUATION_KEY = 'maxent-AD.L-jnt-mst' # TEST_EVALUATION_KEY = 'maxent-AD.L-jnt-eisner' -# TEST_EVALUATION_KEY = 'maxent-iheads-global-AD.L-jnt-eisner' +# TEST_EVALUATION_KEY = 'maxent-AD.L-jnt_su-eisner' +TEST_EVALUATION_KEY = 'maxent-iheads-global-AD.L-jnt-eisner' +# TEST_EVALUATION_KEY = 'maxent-iheads-global-AD.L-jnt_su-eisner' """Evaluation to use for testing. Leave this to None until you think it's OK to look at the test data. @@ -109,8 +120,9 @@ parsed/mrg/wsj) """ -CORENLP_OUT_DIR = None +# CORENLP_OUT_DIR = None # CORENLP_OUT_DIR = '/projets/melodi/corpus/rst-dt-corenlp-2015-01-29' +CORENLP_OUT_DIR = '/home/mmorey/corpora/rst-dt/rst-dt-corenlp-2015-01-29' """ Where to read parses from CoreNLP from """ @@ -126,6 +138,21 @@ Which feature set to use for feature extraction """ +LABEL_SET = 'coarse' # one of {'coarse', 'fine'} or a list of strings +""" +Which label set to use +""" + +SAME_UNIT = 'joint' # one of {'joint', 'preproc', 'no'} +""" +Whether to have a special processing for same-unit +""" + +NARY_ENC = 'tree' # one of {'chain', 'tree'} +""" +Encoding for n-ary nodes in the ctree. +""" + FIXED_FOLD_FILE = None # FIXED_FOLD_FILE = 'folds-TRAINING.json' """ @@ -235,7 +262,7 @@ def _structured(klearner): """ -def _core_parsers(klearner, unique_real_root=True): +def _core_parsers(klearner, unique_real_root=True, same_unit='no'): """Our basic parser configurations """ # joint @@ -253,6 +280,30 @@ def _core_parsers(klearner, unique_real_root=True): use_prob=True)), ] ] + # WIP with same-unit + if same_unit == 'joint': + joint.extend([ + mk_joint_su(klearner, d) for d in [ + # decoder_last(), + # DECODER_LOCAL, + # decoder_mst(), + Keyed('eisner', + EisnerDecoder(unique_real_root=unique_real_root, + use_prob=True)), + ] + ]) + elif same_unit == 'preproc': + joint.extend([ + mk_su_joint(klearner, d) for d in [ + # decoder_last(), + # DECODER_LOCAL, + # decoder_mst(), + Keyed('eisner', + EisnerDecoder(unique_real_root=unique_real_root, + use_prob=True)), + ] + ]) + # end WIP # postlabeling use_prob = klearner.attach.payload.can_predict_proba @@ -261,9 +312,9 @@ def _core_parsers(klearner, unique_real_root=True): # decoder_last() , # DECODER_LOCAL, # decoder_mst(), - Keyed('eisner', - EisnerDecoder(unique_real_root=unique_real_root, - use_prob=use_prob)), + # Keyed('eisner', + # EisnerDecoder(unique_real_root=unique_real_root, + # use_prob=use_prob)), ] ] @@ -296,74 +347,26 @@ def _core_parsers(klearner, unique_real_root=True): HARNESS_NAME = 'irit-rst-dt' -# possibly obsolete -def _mk_basic_intras(klearner, kconf): - """Intra/inter parser based on a single core parser - """ - # NEW intra parsers are explicitly authorized to have more than one - # real root (necessary for the Eisner decoder, maybe other decoders too) - parsers = [IntraInterPair(intra=x, inter=y) for x, y in - zip(_core_parsers(klearner, unique_real_root=False), - _core_parsers(klearner))] - return [combine_intra(p, kconf) for p in parsers] - - -def _mk_sorc_intras(klearner, kconf): - """Intra/inter parsers based on a single core parser - and a sentence oracle - """ - parsers = [IntraInterPair(intra=x, inter=y) for x, y in - zip(_core_parsers(ORACLE, unique_real_root=False), - _core_parsers(klearner))] - return [combine_intra(p, kconf, primary='inter') for p in parsers] - - -def _mk_dorc_intras(klearner, kconf): - """Intra/inter parsers based on a single core parser - and a document oracle - """ - parsers = [IntraInterPair(intra=x, inter=y) for x, y in - zip(_core_parsers(klearner, unique_real_root=False), - _core_parsers(ORACLE))] - return [combine_intra(p, kconf, primary='intra') for p in parsers] - - -def _mk_last_intras(klearner, kconf): - """Parsers using "last" for intra and a core decoder for inter. - """ - if ((not klearner.attach.payload.can_predict_proba or - not klearner.label.payload.can_predict_proba)): - return [] - - kconf = Keyed(key=combined_key('last', kconf), - payload=kconf.payload) - econf_last = mk_joint(klearner, decoder_last()) - parsers = [IntraInterPair(intra=econf_last, inter=y) for y in - _core_parsers(klearner)] - return [combine_intra(p, kconf, primary='inter') for p in parsers] -# end of possibly obsolete - - def _is_junk(econf): """ Any configuration for which this function returns True will be silently discarded """ # intrasential head to head mode only works with mst for now - has = econf.settings - kids = econf.settings.children - has_intra_oracle = has.intra and (kids.intra.oracle or kids.inter.oracle) - has_any_oracle = has.oracle or has_intra_oracle + has_intra_oracle = (econf.settings.intra + and (econf.settings.children.intra.oracle + or econf.settings.children.inter.oracle)) + has_any_oracle = econf.settings.oracle or has_intra_oracle - decoder_name = econf.parser.key[len(has.key) + 1:] + decoder_name = econf.parser.key[len(econf.settings.key) + 1:] # last with last-based intra decoders is a bit redundant - if has.intra and decoder_name == 'last': + if econf.settings.intra and decoder_name == 'last': return True # oracle would be redundant with sentence/doc oracles # FIXME the above is wrong for intra/inter parsers because gold edges # can fall out of the search space - if has.oracle and has_intra_oracle: + if econf.settings.oracle and has_intra_oracle: return True # FIXME should sometimes be False # toggle or comment to enable filtering in/out oracles @@ -378,62 +381,196 @@ def _evaluations(): res = [] # == one-step (global) parsers == - learners = [] - learners.extend(_LOCAL_LEARNERS) - # current structured learners don't do probs, hence non-prob decoders - nonprob_eisner = EisnerDecoder(use_prob=False) - learners.extend(l(nonprob_eisner) for l in _STRUCTURED_LEARNERS) - # MST is disabled by default, as it does not output projective trees - # nonprob_mst = MstDecoder(MstRootStrategy.fake_root, False) - # learners.extend(l(nonprob_mst) for l in _STRUCTURED_LEARNERS) - global_parsers = itr.chain.from_iterable(_core_parsers(l) - for l in learners) - res.extend(global_parsers) + # WIP + # maxent, eisner, AD.L-jnt + maxent_klearner = LearnerConfig(attach=attach_learner_maxent(), + label=label_learner_maxent()) + res.append( + EvaluationConfig( + key='maxent-AD.L-jnt-eisner', + settings=Settings(key='AD.L-jnt', + intra=False, + oracle=False, + children=None), + learner=maxent_klearner, + parser=Keyed('AD.L-jnt-eisner', + JointPipeline( + learner_attach=maxent_klearner.attach.payload, + learner_label=maxent_klearner.label.payload, + decoder=EisnerDecoder(unique_real_root=True, use_prob=True)))) + ) + + # maxent, eisner, AD.L-jnt then overwrite predicted "Same-Unit" + # FIXME "learner" might be wrong: this LearnerConfig has no mention of + # the same-unit classifier + maxent_su_learner = attach_learner_maxent() + # oracle_su_learner = Keyed('oracle', AttachOracle()) # alternative + res.append( + EvaluationConfig( + key='maxent-AD.L-jnt_su-eisner', + settings=Settings(key='AD.L-jnt_su', + intra=False, + oracle=False, + children=None), + # FIXME ("attach", "label"), lacks "same_unit" + learner=maxent_klearner, + parser=Keyed('AD.L-jnt_su-eisner', + JointSameUnitPipeline( + learner_attach=maxent_klearner.attach.payload, + learner_label=maxent_klearner.label.payload, + learner_su=maxent_su_learner.payload, + decoder=EisnerDecoder(unique_real_root=True, use_prob=True)))) + ) + # end WIP + + if False: # legacy code for one-step parsers + learners = [] + learners.extend(_LOCAL_LEARNERS) + # current structured learners don't do probs, hence non-prob decoders + nonprob_eisner = EisnerDecoder(use_prob=False) + learners.extend(l(nonprob_eisner) for l in _STRUCTURED_LEARNERS) + # MST is disabled by default, as it does not output projective trees + # nonprob_mst = MstDecoder(MstRootStrategy.fake_root, False) + # learners.extend(l(nonprob_mst) for l in _STRUCTURED_LEARNERS) + global_parsers = itr.chain.from_iterable( + _core_parsers(l, same_unit=SAME_UNIT) for l in learners) + res.extend(global_parsers) # == two-step parsers: intra then inter-sentential == - ii_learners = [] # (intra, inter) learners - ii_learners.extend((copy.deepcopy(klearner), copy.deepcopy(klearner)) - for klearner in _LOCAL_LEARNERS - if klearner != ORACLE) - # keep pointer to intra and inter oracles - ii_oracles = (copy.deepcopy(ORACLE), ORACLE_INTER) - ii_learners.append(ii_oracles) - # structured learners, cf. supra - intra_nonprob_eisner = EisnerDecoder(use_prob=False, - unique_real_root=True) - inter_nonprob_eisner = EisnerDecoder(use_prob=False, - unique_real_root=True) - ii_learners.extend((copy.deepcopy(l)(intra_nonprob_eisner), - copy.deepcopy(l)(inter_nonprob_eisner)) - for l in _STRUCTURED_LEARNERS) - # couples of learners with either sentence- or document-level oracle - sorc_ii_learners = [ - (ii_oracles[0], inter_lnr) for intra_lnr, inter_lnr in ii_learners - if (ii_oracles[0], inter_lnr) not in ii_learners - ] - dorc_ii_learners = [ - (intra_lnr, ii_oracles[1]) for intra_lnr, inter_lnr in ii_learners - if (intra_lnr, ii_oracles[1]) not in ii_learners - ] - # enumerate pairs of (intra, inter) parsers - ii_pairs = [] - for intra_lnr, inter_lnr in itr.chain(ii_learners, - sorc_ii_learners, - dorc_ii_learners): - # NEW intra parsers are explicitly authorized (in fact, expected) - # to have more than one real root ; this is necessary for the - # Eisner decoder and probably others, with "hard" strategies - ii_pairs.extend(IntraInterPair(intra=x, inter=y) for x, y in - zip(_core_parsers(intra_lnr, unique_real_root=True), # TODO add unique_real_root to hyperparameters in grid search - _core_parsers(inter_lnr, unique_real_root=True))) - # cross-product: pairs of parsers x intra-/inter- configs - ii_parsers = [combine_intra(p, kconf, - primary=('inter' if p.intra.settings.oracle - else 'intra'), - verbose=_VERBOSE_INTRA_INTER) - for p, kconf - in itr.product(ii_pairs, _INTRA_INTER_CONFIGS)] - res.extend(ii_parsers) + # WIP explicit declaration + maxent_team_intra = LearnerConfig(attach=attach_learner_maxent(), + label=label_learner_maxent()) + # FIXME ? maybe sel_inter='global' implies that + # maxent_team_inter = LearnerConfig(attach=maxent_klearner.attach, label=maxent_klearner.label) + maxent_team_inter = LearnerConfig(attach=attach_learner_maxent(), + label=label_learner_maxent()) + res.append( + EvaluationConfig( + key='maxent-iheads-global-AD.L-jnt-eisner', + settings=Settings(key='iheads-global-AD.L-jnt', + intra=True, + oracle=False, + children=IntraInterPair( + intra=Settings(key='AD.L-jnt', + intra=False, + oracle=False, + children=None), + inter=Settings(key='AD.L-jnt', + intra=False, + oracle=False, + children=None))), + learner=IntraInterPair(intra=maxent_team_intra, + inter=maxent_team_inter), + parser=Keyed('iheads-global-AD.L-jnt-eisner', + HeadToHeadParser( + IntraInterPair( + intra=JointPipeline( + learner_attach=maxent_team_intra.attach.payload, + learner_label=maxent_team_intra.label.payload, + decoder=EisnerDecoder(unique_real_root=True, use_prob=True)), + inter=JointPipeline( + learner_attach=maxent_team_inter.attach.payload, + learner_label=maxent_team_inter.label.payload, + decoder=EisnerDecoder(unique_real_root=True, use_prob=True))), + sel_inter='global', + verbose=_VERBOSE_INTRA_INTER))) + ) + + # maxent-iheads-global-AD.L-jnt_su-eisner + maxent_su_learner_intra = attach_learner_maxent() # WIP + res.append( + EvaluationConfig( + key='maxent-iheads-global-AD.L-jnt_su-eisner', + settings=Settings(key='iheads-global-AD.L-jnt_su', + intra=True, + oracle=False, + children=IntraInterPair( + intra=Settings(key='AD.L-jnt_su', + intra=False, + oracle=False, + children=None), + inter=Settings(key='AD.L-jnt', + intra=False, + oracle=False, + children=None))), + learner=IntraInterPair(intra=maxent_team_intra, + inter=maxent_team_inter), + parser=Keyed('iheads-global-AD.L-jnt_su-eisner', + HeadToHeadParser( + IntraInterPair( + intra=JointSameUnitPipeline( + learner_attach=maxent_team_intra.attach.payload, + learner_label=maxent_team_intra.label.payload, + learner_su=maxent_su_learner_intra.payload, + decoder=EisnerDecoder(unique_real_root=True, use_prob=True)), + inter=JointPipeline( + learner_attach=maxent_team_inter.attach.payload, + learner_label=maxent_team_inter.label.payload, + decoder=EisnerDecoder(unique_real_root=True, use_prob=True))), + sel_inter='global', + verbose=_VERBOSE_INTRA_INTER))) + ) + # end WIP + + if False: # disable legacy code for 2-step parsers + ii_learners = [] # (intra, inter) learners + ii_learners.extend((copy.deepcopy(klearner), copy.deepcopy(klearner)) + for klearner in _LOCAL_LEARNERS + if klearner != ORACLE) + # keep pointer to intra and inter oracles + ii_oracles = (copy.deepcopy(ORACLE), ORACLE_INTER) + ii_learners.append(ii_oracles) + # structured learners, cf. supra + intra_nonprob_eisner = EisnerDecoder(use_prob=False, + unique_real_root=True) + inter_nonprob_eisner = EisnerDecoder(use_prob=False, + unique_real_root=True) + + ii_learners.extend((copy.deepcopy(l)(intra_nonprob_eisner), + copy.deepcopy(l)(inter_nonprob_eisner)) + for l in _STRUCTURED_LEARNERS) + # couples of learners with either sentence- or document-level oracle + sorc_ii_learners = [ + (ii_oracles[0], inter_lnr) for intra_lnr, inter_lnr in ii_learners + if (ii_oracles[0], inter_lnr) not in ii_learners + ] + dorc_ii_learners = [ + (intra_lnr, ii_oracles[1]) for intra_lnr, inter_lnr in ii_learners + if (intra_lnr, ii_oracles[1]) not in ii_learners + ] + # enumerate pairs of (intra, inter) parsers + ii_pairs = [] + for intra_lnr, inter_lnr in itr.chain(ii_learners, + sorc_ii_learners, + dorc_ii_learners): + # NEW intra parsers are explicitly authorized (in fact, expected) + # to have more than one real root ; this is necessary for the + # Eisner decoder and probably others, with "hard" strategies + # TODO add unique_real_root to hyperparameters in grid search + intra_parsers = _core_parsers(intra_lnr, unique_real_root=True, + same_unit=SAME_UNIT) + # same-unit is undefined for inter, in the RST-DT corpus + # (at least in our implementation) + inter_parsers = _core_parsers(inter_lnr, unique_real_root=True, + same_unit='no') + if SAME_UNIT != 'no': + # inter_parsers would be twice less numerous than intra_parsers + # => dirty hack: double the inter parsers + inter_parsers = inter_parsers + inter_parsers + + ii_pairs.extend(IntraInterPair(intra=x, inter=y) for x, y + # FIXME should probably not be a zip(), cf dirty hack + # above + in zip(intra_parsers, inter_parsers) + ) + # cross-product: pairs of parsers x intra-/inter- configs + ii_parsers = [combine_intra(p, kconf, + primary=('inter' if p.intra.settings.oracle + else 'intra'), + verbose=_VERBOSE_INTRA_INTER) + for p, kconf + in itr.product(ii_pairs, _INTRA_INTER_CONFIGS)] + res.extend(ii_parsers) return [x for x in res if not _is_junk(x)] @@ -458,9 +595,9 @@ def _want_details(econf): else: learners = [econf.learner] has_maxent = any('maxent' in l.key for l in learners) - has = econf.settings - kids = econf.settings.children - has_intra_oracle = has.intra and (kids.intra.oracle or kids.inter.oracle) + has_intra_oracle = (econf.settings.intra and + (econf.settings.children.intra.oracle or + econf.settings.children.inter.oracle)) return (has_maxent and ('mst' in econf.parser.key or 'astar' in econf.parser.key or 'eisner' in econf.parser.key) and diff --git a/repro/dplp/buildedu.py b/repro/dplp/buildedu.py new file mode 100644 index 0000000..e7517d7 --- /dev/null +++ b/repro/dplp/buildedu.py @@ -0,0 +1,186 @@ +## buildedu.py +## Author: Yangfeng Ji +## Date: 05-03-2015 +## Time-stamp: + +from os import listdir +from os.path import join, basename +from model.classifier import Classifier +from model.docreader import DocReader +from model.sample import SampleGenerator +from cPickle import load +import gzip + + +# MM +from glob import glob +import os + +DOC_EDUS = {os.path.splitext(os.path.basename(f))[0]: f + for f in glob(os.path.join( + '/home/mmorey/melodi/rst/ji_eisenstein', + 'DPLP/data/edus/*/*.edus'))} + + +def load_gold_edus(conll_file): + """Load gold EDUs for injection into a conll file. + + Parameters + ---------- + conll_file: str + Path to the conll file. + + Returns + ------- + edu_idc: list? of int + Index of the EDU for each token. + """ + result = [] # 1 if token is the last of its EDU, 0 otherwise + + doc_name = os.path.splitext(os.path.basename(conll_file))[0] + # find corresponding file with gold EDUs + fname_edus = DOC_EDUS[doc_name] + edus = [] + with open(fname_edus) as f_edus: + for line in f_edus: + line = line.strip() + if not line: + continue + # non-empty line + edus.append(line) + # open conll file and align tokens + edu_idx = 0 + edu_txt = edus[edu_idx] # remaining text of current EDU + with open(conll_file) as f_conll: + for line in f_conll: + line = line.strip() + if not line: + continue + fields = line.split('\t') + wform_conll = fields[2] # word form + # try to read the same amount of characters off the current EDU + wform_edus = edu_txt[:len(wform_conll)] + try: + assert wform_edus == wform_conll + except AssertionError: + if len(wform_edus) < len(wform_conll): + # EDU boundary happens in the middle of a token: + # possible causes: error in the text of the original doc + # (missing whitespace, wrong version of quotes...), or + # a plain error of the segmenter + assert wform_conll.startswith(wform_edus) + # set the EDU boundary at the current token + result.append(1) + # remaining text + rem_txt = wform_conll[len(wform_edus):].strip() + # read the first characters off the next EDU + edu_idx += 1 + if edu_idx == len(edus): + edu_txt = '' + else: + edu_txt = edus[edu_idx] + # read the first characters off the beginning of the + # next EDU, assert that they match + assert edu_txt[:len(rem_txt)] == rem_txt + edu_txt = edu_txt[len(rem_txt):].lstrip() + else: + # we don't know how to handle this (yet) + print(wform_conll, wform_edus) + raise + else: + # print(fields + [edu_idx + 1]) + # update the state of edu_txt for the next iteration + edu_txt = edu_txt[len(wform_conll):].lstrip() + if not edu_txt: + # when the current EDU is exhausted, pass to the next + result.append(1) + edu_idx += 1 + if edu_idx == len(edus): + # normally, the text should be exhausted on both sides + # (.conll and .edus) at the same time ; + # if the .conll has extra text, the following should + # make the assertion above break at the next iteration + # of the loop + edu_txt = '' + else: + edu_txt = edus[edu_idx] + else: + result.append(0) + return result +# end MM + +def main(fmodel, fvocab, rpath, wpath): + clf = Classifier() + dr = DocReader() + clf.loadmodel(fmodel) + flist = [join(rpath,fname) for fname in listdir(rpath) if fname.endswith('conll')] + vocab = load(gzip.open(fvocab)) + for (fidx, fname) in enumerate(flist): + print "Processing file: {}".format(fname) + doc = dr.read(fname, withboundary=False) + # predict segmentation + if False: + sg = SampleGenerator(vocab) + sg.build(doc) + M, _ = sg.getmat() + predlabels = clf.predict(M) + else: + predlabels = load_gold_edus(fname) # RESUME HERE + doc = postprocess(doc, predlabels) + writedoc(doc, fname, wpath) + + +def postprocess(doc, predlabels): + """ Assign predlabels into doc + """ + tokendict = doc.tokendict + for gidx in tokendict.iterkeys(): + if predlabels[gidx] == 1: + tokendict[gidx].boundary = True + else: + tokendict[gidx].boundary = False + if tokendict[gidx].send: + tokendict[gidx].boundary = True + return doc + + +# def writedoc(doc, fname, wpath): +# """ Write doc into a file with the CoNLL-like format +# """ +# tokendict = doc.tokendict +# N = len(tokendict) +# fname = basename(fname) + '.edu' +# fname = join(wpath, fname) +# eduidx = 0 +# with open(fname, 'w') as fout: +# for gidx in range(N): +# fout.write(str(eduidx) + '\n') +# if tokendict[gidx].boundary: +# eduidx += 1 +# if tokendict[gidx].send: +# fout.write('\n') +# print 'Write segmentation: {}'.format(fname) + + +def writedoc(doc, fname, wpath): + """ Write file + """ + tokendict = doc.tokendict + N = len(tokendict) + fname = basename(fname).replace(".conll", ".merge") + fname = join(wpath, fname) + eduidx = 1 + with open(fname, 'w') as fout: + for gidx in range(N): + tok = tokendict[gidx] + line = str(tok.sidx) + "\t" + str(tok.tidx) + "\t" + line += tok.word + "\t" + tok.lemma + "\t" + line += tok.pos + "\t" + tok.deplabel + "\t" + line += str(tok.hidx) + "\t" + tok.ner + "\t" + line += tok.partialparse + "\t" + str(eduidx) + "\n" + fout.write(line) + # Boundary + if tok.boundary: + eduidx += 1 + if tok.send: + fout.write("\n") diff --git a/repro/dplp/rstparser.py b/repro/dplp/rstparser.py new file mode 100644 index 0000000..73b553b --- /dev/null +++ b/repro/dplp/rstparser.py @@ -0,0 +1,32 @@ +## main.py +## Author: Yangfeng Ji +## Date: 09-25-2015 +## Time-stamp: + +from code.evalparser import evalparser +from cPickle import load +import gzip, sys + +def main(path, draw=True): + with gzip.open("resources/bc3200.pickle.gz") as fin: + print 'Load Brown clusters for creating features ...' + bcvocab = load(fin) + evalparser(path=path, report=True, draw=draw, + bcvocab=bcvocab, + withdp=False) + + +if __name__ == '__main__': + if len(sys.argv) == 2: + path = sys.argv[1] + print 'Read files from: {}'.format(path) + main(path) + elif len(sys.argv) == 3: + path = sys.argv[1] + draw = eval(sys.argv[2]) + print 'Read files from {}'.format(path) + main(path, draw) + else: + print "Usage: python rstparser.py file_path [draw_rst_tree]" + print "\tfile_path - path to the segmented file" + diff --git a/repro/gcrf/crf_classifier.py b/repro/gcrf/crf_classifier.py new file mode 100644 index 0000000..58ee1ff --- /dev/null +++ b/repro/gcrf/crf_classifier.py @@ -0,0 +1,87 @@ +import os.path +import subprocess + +import paths + + +class CRFClassifier: + def __init__(self, name, model_type, model_path, model_file, verbose): + self.verbose = verbose + self.name = name + self.type = model_type + self.model_fname = model_file + self.model_path = model_path + + model_fpath = os.path.join(self.model_path, self.model_fname) + if not os.path.exists(model_fpath): + print ('The model path %s for CRF classifier %s does not exist.' + % model_fpath) + raise OSError('Could not create classifier subprocess') + + self.classifier_cmd = [ + '%s/crfsuite-stdin' % paths.CRFSUITE_PATH, + 'tag', '-pi', + '-m', '%s' % model_fpath + ] +# print self.classifier_cmd + self.classifier = subprocess.Popen(self.classifier_cmd, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + + if self.classifier.poll(): + raise OSError('Could not create classifier subprocess, with error info:\n%s' % self.classifier.stderr.readline()) + #self.cnt = 0 + + def classify(self, vectors): +# print '\n'.join(vectors) + "\n\n" + vectors_str = '\n'.join(vectors) + "\n\n" + + lines_out, lines_err = self.classifier.communicate(vectors_str) + + lines = [] + for line in lines_out.split('\n'): + if not line.strip(): + break + lines.append(line) + + # HACKY replace the subprocess closed by communicate() + self.classifier = subprocess.Popen(self.classifier_cmd, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + + if self.classifier.poll(): + raise OSError('Could not create classifier subprocess, with error info:\n%s' % self.classifier.stderr.readline()) + # end HACKY + + if self.classifier.poll(): + raise OSError('crf_classifier subprocess died') + + predictions = [] + for line in lines[1:]: + line = line.strip() +# print line + if line != '': + fields = line.split(':') +# print fields + label = fields[0] + prob = float(fields[1]) + predictions.append((label, prob)) + + seq_prob = float(lines[0].split('\t')[1]) + + return seq_prob, predictions + + def poll(self): + """ + Checks that the classifier processes are still alive + """ + if self.classifier is None: + return True + return self.classifier.poll() is not None + + def unload(self): + if self.classifier is not None and not self.poll(): + self.classifier.stdin.write('\n') + print 'Successfully unloaded %s' % self.name diff --git a/repro/gcrf/environment.yml b/repro/gcrf/environment.yml new file mode 100644 index 0000000..bb816bb --- /dev/null +++ b/repro/gcrf/environment.yml @@ -0,0 +1,4 @@ +name: gcrf +dependencies: + - python=2.7 + - nltk=2.0.4 diff --git a/repro/gcrf/gen_gold_edus.py b/repro/gcrf/gen_gold_edus.py new file mode 100644 index 0000000..4eea6a4 --- /dev/null +++ b/repro/gcrf/gen_gold_edus.py @@ -0,0 +1,179 @@ +"""Generate .edus files for Feng's gCRF parser, with gold EDUs. + +""" + +from __future__ import absolute_import, print_function + +import argparse +from difflib import SequenceMatcher +from glob import glob +import os + +import numpy as np + +TXT_MAP = [ + (' .', '.'), + (' ,', ','), + (' %', '%'), + (' :', ':'), + ('-LRB-', '('), + ('-RRB-', ')'), + # non-breaking space + # FIXME switch to unicode where this is a unique char: u"\u00A0" + ('\xc2\xa0', ' '), + ("do n't", "don't"), + ('...', '. . .'), +] + + +def dump_gcrf_edus_gold(f_gold, f_pred, f_dest): + """Reinject gold segmentation into .edus files output by gCRF. + + Parameters + ---------- + f_gold: str + Path to the gold .edus file + f_pred: str + Path to the predicted .edus file + f_dest: str + Path to the output + """ + txt_gold = f_gold.read() + i_gold = 0 # pointer in txt_gold + + skip_toks = 0 # nb of tokens from _pred that have already been consumed + + for line in f_pred: + tokens_pred = line.split(' ') + # the newline character (marking the end of sentence) is appended + # to the last token + assert tokens_pred[-1][-1] == '\n' + # + for i, tok in enumerate(tokens_pred): + if skip_toks: + # skip tokens from _pred that have already been consumed + skip_toks -= 1 + continue + + while txt_gold[i_gold] == ' ': + # skip whitespaces in gold + i_gold += 1 + + if (tok[0] == '.' and tokens_pred[i - 1][-1] == '.' + and txt_gold[i_gold] != '.'): + # preprocessing adds an extra full stop when the last + # token ends with one (e.g. for abbreviations: + # "Inc." => "Inc. .") + if len(tok) > 1: + # skip extra stop, resume normal matching procedure + tok = tok[1:] + else: + # token is exactly '.' => skip it + continue + + if tok == 'EDU_BREAK': + # predicted EDU break inside sentence + if txt_gold[i_gold] == '\n': + # also in gold => correctly predicted => leave it + print(tok, end=' ', file=f_dest) + i_gold += 1 + continue + else: + # not in gold => erroneously predicted => delete it + # (this is a silent operation) + continue + elif tok == '\n' and txt_gold[i_gold] == '\n': + # happens when the token before the newline was a copy of + # the punctuation added by preprocessing, removed above ; + # ex: "... Inc." => "... Inc. ." + print(tok, end='', file=f_dest) + i_gold += 1 + continue + + if txt_gold[i_gold:i_gold + 5] == '\n ': + # gold EDU break inside sentence, missing from predicted + print('EDU_BREAK', end=' ', file=f_dest) # FIXME to f_dest + i_gold += 5 + + # match token + # whitespaces inside tokens are non-breaking spaces: + # \xc2\xa0 in ascii, but we should really be processing + # them as unicode symbols... + tok_txt_gold = (tok + .replace('\xc2\xa0', ' ') + .replace('-LRB-', '(') + .replace('-RRB-', ')') + .replace('-LCB-', '{') + .replace('-RCB-', '}') + .replace('``', '"') + .replace("''", '"') + .replace('...', '. . .') + ) + if i < len(tokens_pred) - 1: + # all tokens except for the last of the sentence + if (txt_gold[i_gold:i_gold + len(tok_txt_gold)] + == tok_txt_gold): + # it is a match indeed + i_gold += len(tok_txt_gold) + # print token followed by a whitespace + print(tok, end=' ', file=f_dest) # FIXME to f_dest + continue + else: + print() + print('wow') + print(tokens_pred[i:]) + print(repr(txt_gold[i_gold:i_gold + len(tok_txt_gold)]), + repr(tok)) + raise ValueError('gni') + else: + # last token of the sentence + if (txt_gold[i_gold:i_gold + len(tok_txt_gold) + 1] + == tok_txt_gold[:-1] + ' ' + tok[-1]): + # gold has an extra whitespace before the newline + i_gold += len(tok_txt_gold) + 1 + # token but no following whitespace + print(tok, end='', file=f_dest) + elif (txt_gold[i_gold:i_gold + 7] == '. . . .' + and tok == '...\n'): + # pre-processing replaces '[. . .] [.]' with '...' ; + # let's assume it's normal + i_gold += 7 + print(tok, end='', file=f_dest) + else: + print() + print('i-2', tokens_pred[i - 2]) + print('i-1', tokens_pred[i - 1]) + print('i', tokens_pred[i]) + print(repr(txt_gold[i_gold:i_gold + len(tok_txt_gold)]), + repr(tok)) + raise ValueError('pouet') + + +if __name__ == '__main__': + parser = argparse.ArgumentParser( + description='Generate .edus files with gold segmentation') + parser.add_argument('dir_gold', metavar='DIR', + help='folder with the gold files (.edus)') + parser.add_argument('dir_pred', metavar='DIR', + help='folder with the predicted files (.edus)') + parser.add_argument('dir_dest', metavar='DIR', + help='output folder') + + args = parser.parse_args() + + # setup output dir + if not os.path.exists(args.dir_dest): + os.makedirs(args.dir_dest) + + files_edus_gold = sorted(glob(os.path.join(args.dir_gold, '*.edus'))) + files_edus_pred = sorted(glob(os.path.join(args.dir_pred, '*.edus'))) + for file_gold, file_pred in zip(files_edus_gold, files_edus_pred): + print(file_gold) + assert os.path.basename(file_gold) == os.path.basename(file_pred) + file_dest = os.path.join(args.dir_dest, + os.path.basename(file_pred)) + + with open(file_gold) as f_gold: + with open(file_pred) as f_pred: + with open(file_dest, mode='w') as f_dest: + dump_gcrf_edus_gold(f_gold, f_pred, f_dest) diff --git a/repro/gcrf/gold_segmenter.py b/repro/gcrf/gold_segmenter.py new file mode 100644 index 0000000..b963367 --- /dev/null +++ b/repro/gcrf/gold_segmenter.py @@ -0,0 +1,112 @@ +"""Pseudo-segmenter for manual (gold) EDU segmentation. + +""" + +from __future__ import print_function +import os + +import utils.utils + + +class GoldSegmenter(object): + """Gold segmenter""" + + def __init__(self, root, _name='gold_segmenter', verbose=False): + self.root = root # root dir for gold .edu files + self.name = _name + self.verbose = verbose + + def segment(self, doc, filename): + """Segment a document. + + Parameters + ---------- + doc: Document + Internal representation of a document + filename: str + Name of the document + """ + # load true segmentation + doc_predictions = [] + fname_doc = os.path.basename(filename) + fname_edus = os.path.join(self.root, fname_doc + '.edus') + with open(fname_edus) as f_edus: + fedus_sentences = f_edus.readlines() + doc_predictions = [] + for sent in fedus_sentences: + toks = sent.strip().split(' ') + predictions = [] + for tok in toks[:-1]: + if tok == 'EDU_BREAK': + if predictions: + # "not predictions" should not happen, but + # apparently it does, e.g. wsj_1376: + # "EDU_BREAK It provides..." + predictions[-1] = 1 + else: + predictions.append(0) + # set a marginal proba of 1.0 for each prediction + doc_predictions.append([(x, 1.0) for x in predictions]) + + # c/c + doc.edu_word_segmentation = [] + doc.cuts = [] + doc.edus = [] + # end c/c + + for sentence, predictions in zip(doc.sentences, doc_predictions): + self.segment_sentence(sentence, predictions) + + # c/c + doc.start_edu = 0 + doc.end_edu = len(doc.edus) + # end c/c + + def segment_sentence(self, sentence, predictions): + """Segment a sentence. + """ + # c/c from crf_segmenter + if len(sentence.tokens) == 1: + edus = [[sentence.tokens[0].word, sentence.raw_text[-3 : ]]] + + sentence.doc.cuts.append((len(sentence.doc.edus), len(sentence.doc.edus) + len(edus))) + sentence.start_edu = len(sentence.doc.edus) + sentence.end_edu = len(sentence.doc.edus) + len(edus) + sentence.doc.edu_word_segmentation.append([(0, 1)]) + sentence.doc.edus.extend(edus) + return + # end c/c + + # another c/c + edus = [] + edu_word_segmentations = [] + start = 0 + for i in range(len(predictions)): + pred = int(predictions[i][0]) + if pred == 1: +# print i, pred + edu_word_segmentations.append((start, i + 1)) + start = i + 1 + + edu_word_segmentations.append((start, len(sentence.tokens))) + + for (start_word, end_word) in edu_word_segmentations: + edu = [] + for j in range(start_word, end_word): + edu.extend(utils.utils.unescape_penn_special_word(sentence.tokens[j].word).split(' ')) + + if end_word == len(sentence.tokens): +# print sentence.raw_text + edu.append(sentence.raw_text[-3 : ]) + edus.append(edu) + + sentence.doc.cuts.append((len(sentence.doc.edus), len(sentence.doc.edus) + len(edus))) + sentence.start_edu = len(sentence.doc.edus) + sentence.end_edu = len(sentence.doc.edus) + len(edus) + sentence.doc.edu_word_segmentation.append(edu_word_segmentations) + sentence.doc.edus.extend(edus) + # end another c/c + + def unload(self): + """Unload ; a no-op here""" + pass diff --git a/repro/gcrf/parse.py b/repro/gcrf/parse.py new file mode 100644 index 0000000..a0ca48f --- /dev/null +++ b/repro/gcrf/parse.py @@ -0,0 +1,354 @@ +''' +Created on 2014-01-17 + +@author: Vanessa Wei Feng +''' + +import os.path +import sys +import time +import traceback +from datetime import datetime +from optparse import OptionParser + +import paths +import utils.serialize +from document.doc import Document +from logs.log_writer import LogWriter +from prep.preprocesser import Preprocesser +from segmenters.crf_segmenter import CRFSegmenter +from segmenters.gold_segmenter import GoldSegmenter # MM +from treebuilder.build_tree_CRF import CRFTreeBuilder + + +class DiscourseParser(): + def __init__(self, options, output_dir=None, log_writer=None): + self.verbose = options.verbose + self.skip_parsing = options.skip_parsing + self.global_features = options.global_features + self.save_preprocessed_doc = options.save_preprocessed_doc + + self.output_dir = os.path.join( + paths.OUTPUT_PATH, + output_dir if output_dir is not None else '') + if not os.path.exists(self.output_dir): + print 'Output directory %s not exists, creating it now.' % self.output_dir + os.makedirs(self.output_dir) + + self.log_writer = LogWriter(log_writer) + + self.feature_sets = 'gCRF' + + initStart = time.time() + + self.preprocesser = None + try: + self.preprocesser = Preprocesser() + except Exception, e: + print "*** Loading Preprocessing module failed..." + print traceback.print_exc() + + raise e + + # MM enable to load segmentation from .edus files + self.load_edus = options.load_edus + if self.load_edus: + # fake EDU segmenter that loads segmentation from files in a + # folder + self.segmenter = GoldSegmenter(self.load_edus) + else: + try: + self.segmenter = CRFSegmenter( + _name=self.feature_sets, verbose=self.verbose, + global_features=self.global_features) + except Exception, e: + print "*** Loading Segmentation module failed..." + print traceback.print_exc() + raise e + + try: + if not self.skip_parsing: + self.treebuilder = CRFTreeBuilder( + _name=self.feature_sets, verbose=self.verbose) + else: + self.treebuilder = None + except Exception, e: + print "*** Loading Tree-building module failed..." + print traceback.print_exc() + raise e + + initEnd = time.time() + print 'Finished initialization in %.2f seconds.' % (initEnd - initStart) + print + + def unload(self): + if self.preprocesser is not None: + self.preprocesser.unload() + + if not self.segmenter is None: + self.segmenter.unload() + + if not self.treebuilder is None: + self.treebuilder.unload() + + def parse(self, filename): + if not os.path.exists(filename): + print '%s does not exist.' % filename + return + + self.log_writer.write('***** Parsing %s...' % filename) + + try: + core_filename = os.path.split(filename)[1] + serialized_doc_filename = os.path.join(self.output_dir, core_filename + '.doc.ser') + doc = None + if os.path.exists(serialized_doc_filename): + doc = utils.serialize.loadData(core_filename, self.output_dir, '.doc.ser') + + if doc is None or not doc.preprocessed: + preprocessStart = time.time() + doc = Document() + doc.preprocess(filename, self.preprocesser) + + preprocessEnd = time.time() + + print 'Finished preprocessing in %.2f seconds.' % (preprocessEnd - preprocessStart) + self.log_writer.write('Finished preprocessing in %.2f seconds.' % (preprocessEnd - preprocessStart)) + + if self.save_preprocessed_doc: + print 'Saved preprocessed document data to %s.' % serialized_doc_filename + utils.serialize.saveData(core_filename, doc, self.output_dir, '.doc.ser') + + else: + print 'Loaded saved serialized document data.' + + print + except Exception, e: + print "*** Preprocessing failed ***" + print traceback.print_exc() + raise e + + try: + if not doc.segmented: + segStart = time.time() + if self.load_edus: + # MM GoldSegmenter needs a filename + self.segmenter.segment(doc, filename) + else: + self.segmenter.segment(doc) + + if self.verbose: + print 'edus' + for e in doc.edus: + print e + print + print 'cuts' + for cut in doc.cuts: + print cut + print + print 'edu_word_segmentation' + + segEnd = time.time() + print 'Finished segmentation in %.2f seconds.' % (segEnd - segStart) + print 'Segmented into %d EDUs.' % len(doc.edus) + + self.log_writer.write('Finished segmentation in %.2f seconds. Segmented into %d EDUs.' % ((segEnd - segStart), len(doc.edus))) + if self.save_preprocessed_doc: + print 'Saved segmented document data to %s.' % serialized_doc_filename + utils.serialize.saveData(core_filename, doc, self.output_dir, '.doc.ser') + else: + print 'Already segmented into %d EDUs.' % len(doc.edus) + + print + + if options.verbose: + for e in doc.edus: + print e + + except Exception, e: + print "*** Segmentation failed ***" + print traceback.print_exc() + raise e + + try: + ''' Step 2: build text-level discourse tree ''' + if self.skip_parsing: + outfname = os.path.join(self.output_dir, core_filename + ".edus") + print 'Output EDU segmentation result to %s' % outfname + f_o = open(outfname, "w") + for sentence in doc.sentences: + sent_id = sentence.sent_id + edu_segmentation = doc.edu_word_segmentation[sent_id] + i = 0 + sent_out = [] + for (j, token) in enumerate(sentence.tokens): + sent_out.append(token.word) + if j < len(sentence.tokens) - 1 and j == edu_segmentation[i][1] - 1: + sent_out.append('EDU_BREAK') + i += 1 + f_o.write(' '.join(sent_out) + '\n') + + f_o.flush() + f_o.close() + else: + treeBuildStart = time.time() + # + outfname = os.path.join(self.output_dir, core_filename + ".tree") + + pt = self.treebuilder.build_tree(doc) + + print 'Finished tree building.' + + if pt is None: + print "No tree could be built..." + + if not self.treebuilder is None: + self.treebuilder.unload() + + return -1 + + # Unescape the parse tree + if pt: + doc.discourse_tree = pt + treeBuildEnd = time.time() + + # print out + print 'Finished tree building in %.2f seconds.' % (treeBuildEnd - treeBuildStart) + self.log_writer.write('Finished tree building in %.2f seconds.' % (treeBuildEnd - treeBuildStart)) + + for i in range(len(doc.edus)): + pt.__setitem__(pt.leaf_treeposition(i), '_!%s!_' % ' '.join(doc.edus[i])) + + out = pt.pprint() + print 'Output tree building result to %s.' % outfname + f_o = open(outfname, "w") + f_o.write(out) + f_o.close() + + + if self.save_preprocessed_doc: + print 'Saved fully processed document data to %s.' % serialized_doc_filename + utils.serialize.saveData(core_filename, doc, self.output_dir, '.doc.ser') + + print + except Exception, e: + print traceback.print_exc() + + raise e + + print '===================================================' + #return dists#, probs + +def main(options, args): + parser = None + try: + if options.output_dir: + output_dir = args[0] + start_arg = 1 + else: + output_dir = None + start_arg = 0 + + log_writer = None + if options.logging: + log_fname = os.path.join(paths.LOGS_PATH, 'log_%s.txt' % (output_dir if output_dir else datetime.now().strftime('%Y_%m_%d_%H_%M_%S'))) + log_writer = open(log_fname, 'w') + + if options.filelist: + file_fname = args[start_arg] + if not os.path.exists(file_fname) or not os.path.isfile(file_fname): + print 'The specified file list %s is not a file or does not exist' % file_fname + return + + parser = DiscourseParser(options = options, + output_dir = output_dir, + log_writer = log_writer) + + files = [] + skips = 0 + if options.filelist: + file_fname = args[start_arg] + for line in open(file_fname).readlines(): + fname = line.strip() + + if os.path.exists(fname): + if os.path.exists(os.path.join(parser.output_dir, os.path.split(fname)[1] + '.tree')): + skips += 1 + else: + files.append(fname) + else: + skips += 1 +# print 'Skip %s since it does not exist.' % fname + else: + fname = args[start_arg] +# print os.path.join(paths.tmp_folder, os.path.split(fname)[1] + '.xml') + if os.path.exists(fname): + if os.path.exists(os.path.join(parser.output_dir, os.path.split(fname)[1] + '.tree')): + skips += 1 + else: + files.append(fname) + else: + skips += 1 + + print 'Processing %d documents, skipping %d' % (len(files), skips) + + for (i, filename) in enumerate(files): + print 'Parsing %s, progress: %.2f (%d out of %d)' % (filename, i * 100.0 / len(files), i, len(files)) + + try: + parser.parse(filename) + + parser.log_writer.write('===================================================') + except Exception, e: + print 'Some error occurred, skipping the file' + raise e + + parser.unload() + + except Exception, e: + print traceback.print_exc() + if not parser is None: + parser.unload() + + +v = '1.0' +if __name__ == '__main__': + usage = "Usage: %prog [options] input_file/dir" + + optParser = OptionParser(usage=usage, version="%prog " + v) + optParser.add_option("-v", "--verbose", + action="store_true", dest="verbose", default=False, + help="verbose mode") + optParser.add_option("-s", "--skip_parsing", + action="store_true", dest="skip_parsing", default=False, + help="Skip parsing, i.e., conduct segmentation only.") + optParser.add_option("-D", "--filelist", + action="store_true", dest="filelist", default=False, + help="parse all files specified in the filelist file, one file per line.") + optParser.add_option("-t", "--output_dir", + action="store_true", dest="output_dir", default=False, + help="Specify a directory for output files.") + optParser.add_option("-g", "--global_features", + action="store_true", dest="global_features", default=False, + help="Perform a second pass of EDU segmentation using global features.") + optParser.add_option("-l", "--logging", + action="store_true", dest="logging", default=False, + help="Perform logging while parsing.") + optParser.add_option("-e", "--save", + action="store_true", dest="save_preprocessed_doc", + default=False, + help="Save preprocessed document into serialized file for future use.") + # MM add option to load segmentation from the .edus files that result + # from calling this parser with the --skip_parsing option + optParser.add_option('-r', '--load_edus', + dest='load_edus', default=False, + help="Read segmentation from .edus files in folder") + # end MM + + (options, args) = optParser.parse_args() + if len(args) == 0: + optParser.print_help() + sys.exit(1) + + main(options, args) + diff --git a/repro/gcrf/preprocesser.py b/repro/gcrf/preprocesser.py new file mode 100644 index 0000000..0d5be7b --- /dev/null +++ b/repro/gcrf/preprocesser.py @@ -0,0 +1,240 @@ +''' +Created on 2014-01-18 + +@author: Wei +''' +import subprocess +import paths +from document.sentence import Sentence +from document.token import Token +from trees.lexicalized_tree import LexicalizedTree +import prep_utils +import os.path +from syntax_parser import SyntaxParser +from document.dependency import Dependency +import re + +class Preprocesser: + def __init__(self): + self.syntax_parser = None + + try: + self.syntax_parser = SyntaxParser() + except Exception, e: + raise e + + self.max_sentence_len = 100 + + def heuristic_sentence_splitting(self, raw_sent): + if len(raw_sent) == 0: + return [] + + if len(raw_sent.split()) <= self.max_sentence_len: + return [raw_sent] + + i = len(raw_sent) / 2 + j = i + k = i + 1 + boundaries = [';', ':', '!', '?'] + + results = [] + while j > 0 and k < len(raw_sent) - 1: + if raw_sent[j] in boundaries: + l_sent = raw_sent[ : j + 1] + r_sent = raw_sent[j + 1 : ].strip() + + if len(l_sent.split()) > 1 and len(r_sent.split()) > 1: + results.extend(self.heuristic_sentence_splitting(l_sent)) + results.extend(self.heuristic_sentence_splitting(r_sent)) + return results + else: + j -= 1 + k += 1 + elif raw_sent[k] in boundaries: + l_sent = raw_sent[ : k + 1] + r_sent = raw_sent[k + 1 : ].strip() + + if len(l_sent.split()) > 1 and len(r_sent.split()) > 1: + results.extend(self.heuristic_sentence_splitting(l_sent)) + results.extend(self.heuristic_sentence_splitting(r_sent)) + return results + else: + j -= 1 + k += 1 + else: + j -= 1 + k += 1 + + if len(results) == 0: + return [raw_sent] + + + def parse_single_sentence(self, raw_text): + return self.syntax_parser.parse_sentence(raw_text) + + + def process_single_sentence(self, doc, raw_text, end_of_para): + sentence = Sentence(len(doc.sentences), raw_text + ('' if not end_of_para else '

'), doc) + parse_tree_str, deps_str = self.parse_single_sentence(raw_text) + + parse = LexicalizedTree.parse(parse_tree_str, leaf_pattern = '(?<=\\s)[^\)\(]+') + sentence.set_unlexicalized_tree(parse) + + for (token_id, te) in enumerate(parse.leaves()): + word = te + token = Token(word, token_id + 1, sentence) + sentence.add_token(token) + + heads = self.get_heads(sentence, deps_str.split('\n')) + sentence.heads = heads + sentence.set_lexicalized_tree(prep_utils.create_lexicalized_tree(parse, heads)) + + doc.add_sentence(sentence) + + + def get_heads(self, sentence, dep_elems): + heads = [] + for token in sentence.tokens: + heads.append([token.word, token.get_PoS_tag(), 0]) + + for dep_e in dep_elems: + m = re.match('(.+?)\((.+?)-(\d+?), (.+?)-(\d+?)\)', dep_e) + if m: + relation = m.group(1) + gov_id = int(m.group(3)) + dep_id = int(m.group(5)) + + heads[dep_id - 1][2] = gov_id + sentence.add_dependency(Dependency(gov_id, dep_id, relation)) + + + return heads + + + def sentence_splitting(self, raw_filename, doc): + doc.sentences = [] + + cmd = 'perl %s/boundary.pl -d %s/HONORIFICS -i %s' % (paths.SSPLITTER_PATH, paths.SSPLITTER_PATH, os.path.abspath(raw_filename)) + + p = subprocess.Popen(cmd, stdout = subprocess.PIPE, stderr = subprocess.PIPE, shell = True) + output, errdata = p.communicate() + + if len(errdata) == 0: + raw_paras = output.strip().split('\n\n') + seg_sents = [] + for para_idx, raw_string in enumerate(raw_paras): + raw_sentences = raw_string.split('\n') + # MM + if (os.path.basename(raw_filename) == 'wsj_0655.out' + and para_idx == 8): + # this error is in the original text *and* is redone + # by the segmenter: + # the segmenter wrongly splits on "[{Mr.] [Ortega's}]" + # => repair by merging sentences + raw_sentences = ([raw_sentences[0] + ' ' + + raw_sentences[1]] + + raw_sentences[2:]) + elif (os.path.basename(raw_filename) == 'wsj_1169.out' + and para_idx == 0): + # this error is in the original text *and* is redone + # by the segmenter: + # "[Murata Mfg.] [Co.]" + raw_sentences = ([raw_sentences[0] + ' ' + + raw_sentences[1]] + + raw_sentences[2:]) + elif (os.path.basename(raw_filename) == 'wsj_1169.out' + and para_idx == 2): + # this error is in the original text *and* is redone + # by the segmenter: + # [G.m.b.] [H.] + raw_sentences = ([raw_sentences[0] + ' ' + + raw_sentences[1]] + + raw_sentences[2:]) + elif (os.path.basename(raw_filename) == 'wsj_1331.out' + and para_idx == 9): + # text is correct, only the segmenter makes an error: + # [all over again.] ['"] + raw_sentences = (raw_sentences[:1] + + [raw_sentences[1] + ' ' + + raw_sentences[2]]) + elif (os.path.basename(raw_filename) == 'wsj_1376.out' + and para_idx == 5): + # text is correct, only the segmenter makes an error: + # [society.] [. . .] + raw_sentences = (raw_sentences[:1] + + [raw_sentences[1] + ' ' + + raw_sentences[2]] + + raw_sentences[3:]) + elif (os.path.basename(raw_filename) == 'wsj_1376.out' + and para_idx == 6): + # [` Hello.] ['] (twice) + # move the trailing "'" up from the next raw sentence, + # and drop the whitespace after it + raw_sentences[3] = raw_sentences[3] + raw_sentences[4][0] + raw_sentences[4] = raw_sentences[4][2:] + # same for the next sentence + raw_sentences[4] = raw_sentences[4] + raw_sentences[5][0] + raw_sentences[5] = raw_sentences[5][2:] + elif (os.path.basename(raw_filename) == 'wsj_1376.out' + and para_idx == 21): + # error by the segmenter + raw_sentences[0] = (raw_sentences[0] + ' ' + + raw_sentences[1] + ' ' + + raw_sentences[2]) + raw_sentences = raw_sentences[:1] + raw_sentences[3:] + elif (os.path.basename(raw_filename) == 'wsj_1380.out' + and para_idx == 6): + # error by the segmenter + # [... Boston Inc. .] ['s First ...] + raw_sentences[0] = (raw_sentences[0] + ' ' + + raw_sentences[1]) + raw_sentences = raw_sentences[:1] + elif (os.path.basename(raw_filename) == 'wsj_2385.out' + and para_idx in [4, 5, 12]): + # error by the segmenter + # double dash is equivalent here to ":", hence same + # sentence, ex: [... Co. .][-- ...] + raw_sentences[0] = (raw_sentences[0] + ' ' + + raw_sentences[1]) + raw_sentences = raw_sentences[:1] + elif (os.path.basename(raw_filename) == 'wsj_2386.out' + and para_idx == 2): + # error by the segmenter + raw_sentences[0] = (raw_sentences[0] + ' ' + + raw_sentences[1]) + raw_sentences = raw_sentences[:1] + raw_sentences[2:] + elif False: + print para_idx + print raw_sentences + # end MM + for (i, raw_sent) in enumerate(raw_sentences): + if len(raw_sent.split()) > self.max_sentence_len: + chunked_raw_sents = self.heuristic_sentence_splitting(raw_sent) + if len(chunked_raw_sents) == 1: + continue + + for (j, sent) in enumerate(chunked_raw_sents): + seg_sents.append((sent, i == len(raw_sentences) - 1 and j == len(chunked_raw_sents))) + else: + seg_sents.append((raw_sent, i == len(raw_sentences) - 1)) + # MM + if False and os.path.basename(raw_filename) == 'wsj_2386.out': + raise ValueError('gni') + # end MM + else: + raise NameError("*** Sentence splitter crashed, with trace %s..." % errdata) + + + for (i, (raw_text, end_of_para)) in enumerate(seg_sents): + if i % 10 == 0: + print 'Processing sentence %d out of %d' % (i, len(seg_sents)) + + self.process_single_sentence(doc, raw_text, end_of_para) + + def preprocess(self, raw_filename, doc): + self.sentence_splitting(raw_filename, doc) + + + def unload(self): + if self.syntax_parser: + self.syntax_parser.unload() diff --git a/requirements.txt b/requirements.txt index 7d348b6..4735983 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,6 @@ --e git+https://github.com/irit-melodi/educe.git#egg=educe --e git+https://github.com/irit-melodi/attelo.git#egg=attelo --e git+https://github.com/nlhepler/pydot.git#egg=pydot +# -e git+https://github.com/irit-melodi/educe.git#egg=educe +-e /home/mmorey/melodi/educe +# -e git+https://github.com/irit-melodi/attelo.git#egg=attelo +-e /home/mmorey/melodi/attelo +# -e git+https://github.com/nlhepler/pydot.git#egg=pydot -e .