diff --git a/attelo/args.py b/attelo/args.py index 403714d..af463de 100644 --- a/attelo/args.py +++ b/attelo/args.py @@ -24,6 +24,8 @@ def add_common_args(psr): help="EDU pair features (libsvm)") psr.add_argument("vocab", metavar="FILE", help="feature vocabulary") + psr.add_argument("labels", metavar="FILE", + help="labels") psr.add_argument("--quiet", action="store_true", help="Supress all feedback") diff --git a/attelo/cdu.py b/attelo/cdu.py new file mode 100644 index 0000000..28d4995 --- /dev/null +++ b/attelo/cdu.py @@ -0,0 +1,13 @@ +"""Explicit representation of a CDU. + +As of 2016-07-28, this is WIP. +""" + +from collections import namedtuple + + +class CDU(namedtuple("CDU", "id members")): + """A class representing the CDU (id, [members])""" + pass + + diff --git a/attelo/cmd/graph.py b/attelo/cmd/graph.py index 08d9346..f16d6e5 100644 --- a/attelo/cmd/graph.py +++ b/attelo/cmd/graph.py @@ -29,9 +29,9 @@ def config_argparser(psr): input_grp = psr.add_mutually_exclusive_group(required=True) input_grp.add_argument("--gold", metavar="FILE", - nargs=2, + nargs=3, help="gold predictions [pairings, " - "features (targets only)]") + "features (targets only), labels]") input_grp.add_argument("--predictions", metavar="FILE", help="single predictions") diff --git a/attelo/cmd/util.py b/attelo/cmd/util.py index 2b8b9f7..eb7b7d9 100644 --- a/attelo/cmd/util.py +++ b/attelo/cmd/util.py @@ -15,10 +15,10 @@ def load_args_multipack(args): ''' Load multipack specified via command line arguments ''' - return load_multipack(args.edus, - args.pairings, + return load_multipack(args.edus, args.pairings, args.features, - args.vocab, + args.vocab, args.labels, + file_split='corpus', # WIP verbose=not args.quiet) diff --git a/attelo/decoding/eisner.py b/attelo/decoding/eisner.py index d0fc662..2ebb8af 100644 --- a/attelo/decoding/eisner.py +++ b/attelo/decoding/eisner.py @@ -3,6 +3,7 @@ import numpy as np +from ..edu import FAKE_ROOT from .interface import Decoder # temporary? imports from ..table import _edu_positions @@ -46,6 +47,9 @@ def decode(self, dpack, nonfixed_pairs=None): """ # whether the output tree should contain a unique real root unique_real_root = self._unique_real_root + # check that the first EDU is the fake root ; this is an + # important assumption for the following code + assert dpack.edus[0] == FAKE_ROOT # get number of EDUs and possible labels nb_edus = len(dpack.edus) diff --git a/attelo/edu.py b/attelo/edu.py index 105e470..f1ef39e 100644 --- a/attelo/edu.py +++ b/attelo/edu.py @@ -53,3 +53,12 @@ def span(self): all groupings """ # pylint: enable=pointless-string-statement + + +# small helper for parsers +def edu_id2num(edu_id): + """Get the number of an EDU""" + edu_num = (int(edu_id.rsplit('_', 1)[1]) + if edu_id != FAKE_ROOT_ID + else 0) + return edu_num diff --git a/attelo/fold.py b/attelo/fold.py index 47d3b8b..848cb88 100644 --- a/attelo/fold.py +++ b/attelo/fold.py @@ -1,13 +1,13 @@ -''' +""" Group-aware n-fold evaluation. Attelo uses a variant of n-fold evaluation, where we (still) -andomly partition the dataset into a set of folds of roughly even +randomly partition the dataset into a set of folds of roughly even size, but respecting the additional constraint that any two data -entries belonging in the same "group" (determined a single +entries belonging in the same "group" (determined by a single distiguished feature, eg. the document id, the dialogue id, etc) are always in the same fold. Note that this makes it a bit harder -to have perfectly evenly sized folds +to have perfectly evenly sized folds. Created on Jun 20, 2012 @@ -15,7 +15,12 @@ @author: stergos contribs: phil -''' + +TODO +---- +* [ ] refactor after `sklearn.model_selection._split`: encapsulate + into a class similar to GroupKFold. +""" import random diff --git a/attelo/harness/evaluate.py b/attelo/harness/evaluate.py index c5da451..fc5cf1d 100644 --- a/attelo/harness/evaluate.py +++ b/attelo/harness/evaluate.py @@ -78,6 +78,18 @@ def _link_data_files(data_dir, eval_dir): eval_file = fp.join(eval_dir, fname) if fp.isfile(data_file) and not fp.exists(eval_file): os.link(data_file, eval_file) + elif fp.isdir(data_file) and not fp.exists(eval_file): + # 2016-09-01 add support for one file per doc: + # create hard links to data/Y/Z as eval-xxx/Y/Z ; + # folders cannot be hard linked so we create copies + os.makedirs(eval_file) + # dirty recursive calls, limited to the immediate members + # of data/ + for fname_sub in os.listdir(data_file): + data_file_sub = fp.join(data_file, fname_sub) + eval_file_sub = fp.join(eval_file, fname_sub) + if fp.isfile(data_file_sub) and not fp.exists(eval_file_sub): + os.link(data_file_sub, eval_file_sub) def _link_model_files(old_dir, new_dir): @@ -131,12 +143,26 @@ def _create_tstamped_dir(prefix, suffix): return True -def prepare_dirs(runcfg, data_dir): - """ - Return eval and scratch directory paths +def prepare_dirs(runcfg, base_dir): + """Get eval and scratch directory paths. + + Parameters + ---------- + runcfg: attelo.harness.config.RuntimeConfig + Current runtime config + base_dir: filepath + Base directory for the experiment. + + Returns + ------- + eval_dir: filepath + Evaluation folder ; subfolder of base_dir. + scratch_dir: filepath + Scratch folder ; subfolder of base_dir. """ - eval_prefix = fp.join(data_dir, "eval") - scratch_prefix = fp.join(data_dir, "scratch") + data_dir = os.path.join(base_dir, 'data') + eval_prefix = fp.join(base_dir, "eval") + scratch_prefix = fp.join(base_dir, "scratch") eval_current = eval_prefix + '-current' scratch_current = scratch_prefix + '-current' @@ -230,11 +256,17 @@ def _load_harness_multipack(hconf, test_data=False): paths = stripped_paths else: paths = hconf.mpack_paths(test_data, stripped=False) - mpack = load_multipack(paths['edu_input'], - paths['pairings'], + mpack = load_multipack(paths['edu_input'], paths['pairings'], paths['features'], - paths['vocab'], - corpus_path=paths.get('corpus', None), # WIP + paths['vocab'], paths['labels'], + # WIP additional files, used only for rst-dt + # as of 2016-07-28 + cdu_file=paths.get('cdu_input', None), + cdu_pairings_file=paths.get('cdu_pairings', None), + cdu_feature_file=paths.get('cdu_features', None), + corpus_path=paths.get('corpus', None), + # end WIP + file_split='corpus', # WIP verbose=True) return mpack @@ -242,7 +274,15 @@ def _load_harness_multipack(hconf, test_data=False): def _init_corpus(hconf): """Start evaluation; generate folds if needed - :rtype: DataConfig or None + Parameters + ---------- + hconf: ?? + TODO + + Returns + ------- + dconf: DataConfig or None + Data configuration """ can_skip_folds = fp.exists(hconf.fold_file) msg_skip_folds = ('Skipping generation of fold files ' @@ -281,8 +321,8 @@ def evaluate_corpus(hconf): dconf = _init_corpus(hconf) if hconf.runcfg.stage in [None, ClusterStage.main]: - foldset = hconf.runcfg.folds if hconf.runcfg.folds is not None\ - else frozenset(dconf.folds.values()) + foldset = (hconf.runcfg.folds if hconf.runcfg.folds is not None + else frozenset(dconf.folds.values())) for fold in foldset: do_fold(hconf, dconf, fold) diff --git a/attelo/harness/example.py b/attelo/harness/example.py index f2cea32..064595f 100644 --- a/attelo/harness/example.py +++ b/attelo/harness/example.py @@ -59,16 +59,16 @@ class TinyHarness(Harness): parser=_parser2)] def __init__(self): - self._datadir = mkdtemp() + self._basedir = mkdtemp() for cpath in glob.glob('doc/example-corpus/*'): - shutil.copy(cpath, self._datadir) + shutil.copy(cpath, self._basedir) super(TinyHarness, self).__init__('tiny', None) def run(self): """Run the evaluation """ runcfg = RuntimeConfig.empty() - eval_dir, scratch_dir = prepare_dirs(runcfg, self._datadir) + eval_dir, scratch_dir = prepare_dirs(runcfg, self._basedir) self.load(runcfg, eval_dir, scratch_dir) evaluate_corpus(self) @@ -89,13 +89,12 @@ def mpack_paths(self, _, stripped=False): The 2nd argument denoted by '_' is test_data, which is unused in this example. """ - core_path = fp.join(self._datadir, 'tiny') - return { - 'edu_input': core_path + '.edus', - 'pairings': core_path + '.pairings', - 'features': core_path + '.features.sparse', - 'vocab': core_path + '.features.sparse.vocab' - } + core_path = fp.join(self._basedir, 'data', 'tiny') + return {'edu_input': core_path + '.edus', + 'pairings': core_path + '.pairings', + 'features': core_path + '.features.sparse', + 'vocab': core_path + '.features.sparse.vocab', + 'labels': core_path + '.labels'} def _model_basename(self, rconf, mtype, ext): "Basic filename for a model" diff --git a/attelo/harness/graph.py b/attelo/harness/graph.py index 46993a2..2ba3a73 100644 --- a/attelo/harness/graph.py +++ b/attelo/harness/graph.py @@ -50,20 +50,19 @@ def _mk_econf_graphs(hconf, edus, gold, econf, fold): raise Exception('Unknown diff mode {}'.format(diffmode)) want_test = fold is None - suffix = 'test' if want_test\ - else fp.basename(hconf.fold_dir_path(fold)) + suffix = ('test' if want_test + else fp.basename(hconf.fold_dir_path(fold))) output_dir = fp.join(hconf.report_dir_path(want_test, None), output_bn_prefix + suffix, econf.key) # settings to_hide = 'inter' if diffmode == GraphDiffMode.diff_intra else None - settings =\ - GraphSettings(hide=to_hide, - select=hconf.graph_docs, - unrelated=False, - timeout=15, - quiet=False) + settings = GraphSettings(hide=to_hide, + select=hconf.graph_docs, + unrelated=False, + timeout=15, + quiet=False) if diffmode == GraphDiffMode.solo: yield delayed(graph_all)(edus, @@ -84,12 +83,11 @@ def _mk_gold_graphs(hconf, dconf): output_dir = fp.join(hconf.report_dir_path(None), 'graphs-gold') - settings =\ - GraphSettings(hide=None, - select=hconf.graph_docs, - unrelated=False, - timeout=15, - quiet=True) + settings = GraphSettings(hide=None, + select=hconf.graph_docs, + unrelated=False, + timeout=15, + quiet=True) predictions = to_predictions(dconf.pack) edus = concat_l(dpack.edus for dpack in dconf.pack.values()) diff --git a/attelo/harness/interface.py b/attelo/harness/interface.py index 5a20a4a..8a4cf70 100644 --- a/attelo/harness/interface.py +++ b/attelo/harness/interface.py @@ -201,8 +201,8 @@ def mpack_paths(self, test_data, stripped=False): Usual keys are: * edu_input * pairings - * features - * vocab + * vocabulary + * labels Parameters ---------- diff --git a/attelo/harness/report.py b/attelo/harness/report.py index 6576fff..e1125f9 100644 --- a/attelo/harness/report.py +++ b/attelo/harness/report.py @@ -309,7 +309,11 @@ def full_report(mpack, fold_dict, slices, metrics, edge_count[key].append(score_edges(fpack, predictions)) # * on constituency tree spans if 'cspans' in metrics: - sc_cspans = score_cspans(dpacks, dpredictions) + try: + sc_cspans = score_cspans(dpacks, dpredictions) + except Exception: + print('Error in slice configuration', key) + raise cspan_count[key].append(sc_cspans) # * on EDUs if 'edus' in metrics: diff --git a/attelo/io.py b/attelo/io.py index 84a5463..08762b3 100644 --- a/attelo/io.py +++ b/attelo/io.py @@ -5,9 +5,12 @@ from __future__ import print_function from itertools import chain import codecs +from collections import defaultdict import copy import csv +from glob import glob import json +import os import sys import time import traceback @@ -16,6 +19,7 @@ import educe # WIP +from .cdu import CDU from .edu import (EDU, FAKE_ROOT_ID, FAKE_ROOT) from .table import (DataPack, DataPackException, UNKNOWN, UNRELATED, @@ -143,6 +147,27 @@ def read_edu(row): return [read_edu(r) for r in reader if r] +def load_cdus(cdu_file): + """Load the description of CDUs. + + As of 2016-07-28, this is WIP. + + Parameters + ---------- + cdu_file : pathname + Path to a file that describes CDUs. Each line provides the + identifier of the CDU, then the list of its member DUs. + + Returns + ------- + cdus : list of CDU + CDUs built from the file content. + """ + with open(cdu_file, 'rb') as instream: + reader = csv.reader(instream, dialect=csv.excel_tab) + return [CDU(x[0], tuple(x[1:])) for x in reader if x] + + def load_pairings(edu_file): """ Read and return EDU pairings (see :doc:`../input`). @@ -167,21 +192,21 @@ def read_pair(row): return [read_pair(r) for r in reader if r] -def load_labels(feature_file): - """ - Read the very top of a feature file and read the labels comment, - return the sequence of labels, else return None +def load_labels(labels_file): + """Read the list of labels. - :rtype: [string] or None + Returns + ------- + labels: list of strings + List of labels """ - with codecs.open(feature_file, 'r', 'utf-8') as stream: - line = stream.readline() - if line.startswith('#'): - seq = line[1:].split() - if seq[0] == 'labels:': - return seq[1:] - # fall-through case, no labels found - return None + lbl_map = dict() + with codecs.open(labels_file, 'r', 'utf-8') as stream: + for line in stream: + i, lbl = line.strip().split() + lbl_map[lbl] = int(i) + labels = [lbl for lbl, i in sorted(lbl_map.items(), key=lambda x: x[1])] + return labels def _process_edu_links(edus, pairings): @@ -212,8 +237,150 @@ def _process_edu_links(edus, pairings): return edus2, pairings2 +def _process_cdu_links(cdus, edus, pairings): + """Convert from the results of `load_cdus` and `load_pairings` to a + sequence of CDUs and pairings respectively. + + Parameters + ---------- + cdus : list of CDU + CDUs + edus : list of EDU + EDUs + pairings : list of pairs of string + List of pairings from/to CDUs + + Returns + ------- + cdus : list of CDU + CDUs + pairings : list of pairs of (EDU or CDU, EDU or CDU) + List of pairs of DUs + """ + du_map = {e.id: e for e in edus} + du_map.update({x.id: x for x in cdus}) + + du_names = frozenset(chain.from_iterable(pairings)) + + naughty = [x for x in du_names if x not in du_map] + if naughty: + oops = ('The pairings file mentions the following DUs but the EDU ' + 'and CDU files do not actually include DUs to go with them:' + ' {}') + raise DataPackException(oops.format(truncate(', '.join(naughty), + 1000))) + pairings = [(du_map[src], du_map[tgt]) for src, tgt in pairings] + + return cdus, pairings + + +def _load_multipack_cdus(cdu_file, cdu_pairings_file, cdu_feature_file, + vocab, doc_names, verbose=False): + """Helper to load the CDU part of a multipack. + + Parameters + ---------- + cdu_file : str + CDU file + + cdu_pairings_file : str + CDU pairings file + + cdu_feature_file : str + Feature file for CDU pairings + + vocab : dict(str, int)? + Feature vocabulary + + doc_names : :obj:`list` of str + List of document names. + + Returns + ------- + doc_cdus : dict(str, TODO) + Map document names to CDUs + + doc_cdu_pairings : dict(str, TODO) + Map document names to CDU pairings + + doc_cdu_data : dict(str, TODO) + Map document names to CDU data (features). + + doc_cdu_targets : dict(str, TODO) + Map document names to CDU targets. + """ + if (cdu_file is not None + and cdu_pairings_file is not None + and cdu_feature_file is not None): + # WIP one file per doc + cdu_files = {os.path.basename(f).rsplit('.', 4)[0]: f + for f in glob(cdu_file)} + cdu_pairings_files = {os.path.basename(f).rsplit('.', 4)[0]: f + for f in glob(cdu_pairings_file)} + cdu_feature_files = {os.path.basename(f).rsplit('.', 3)[0]: f + for f in glob(cdu_feature_file)} + else: + cdu_files = None + cdu_pairings_files = None + cdu_feature_files = None + + doc_cdus = dict() + doc_cdu_pairings = dict() + doc_cdu_data = dict() + doc_cdu_targets = dict() + if (cdu_files is not None + and cdu_pairings_files is not None + and cdu_feature_files is not None): + # augment DataPack with CDUs and pairings from/to them + with Torpor("Reading CDUs and pairings", quiet=not verbose): + for doc_name in doc_names: + # one file per doc + cdu_f = cdu_files[doc_name] + cdu_pairings_f = cdu_pairings_files[doc_name] + # end one file per doc + cdus, cdu_pairings = _process_cdu_links( + load_cdus(cdu_f), + edus, + load_pairings(cdu_pairings_f)) + doc_cdus[doc_name] = cdus + doc_cdu_pairings[doc_name] = cdu_pairings + + with Torpor("Reading features for CDU pairings", quiet=not verbose): + for doc_name in doc_names: + cdu_feature_f = cdu_feature_files[doc_name] + if doc_cdu_pairings[doc_name]: + # CDU files use the same label set as the EDU files ; + # this is not really enforced, but it is implemented + # this way in irit_rst_dt.cmd.gather + # pylint: disable=unbalanced-tuple-unpacking + cdu_data, cdu_targets = load_svmlight_file( + cdu_feature_f, n_features=len(vocab)) + else: + cdu_data = None + cdu_targets = None + # pylint: enable=unbalanced-tuple-unpacking + doc_cdu_data[doc_name] = cdu_data + doc_cdu_targets[doc_name] = cdu_targets + else: + cdus = None + cdu_pairings = None + cdu_data = None + cdu_targets = None + # build dictionaries for CDU stuff + doc_cdus = {doc_name: None for doc_name in doc_names} + doc_cdu_pairings = {doc_name: None for doc_name in doc_names} + doc_cdu_data = {doc_name: None for doc_name in doc_names} + doc_cdu_targets = {doc_name: None for doc_name in doc_names} + # end WIP CDUs + return tuple([doc_cdus, doc_cdu_pairings, doc_cdu_data, doc_cdu_targets]) + + def load_multipack(edu_file, pairings_file, feature_file, vocab_file, + labels_file, + cdu_file=None, cdu_pairings_file=None, + cdu_feature_file=None, corpus_path=None, # WIP + file_split='corpus', # WIP verbose=False): """Read EDUs and features for edu pairs. @@ -224,28 +391,33 @@ def load_multipack(edu_file, pairings_file, feature_file, vocab_file, ---------- ... TODO - corpus_path : string + corpus_path : str Path to the labelled corpus, to retrieve the original gold structures ; at the moment, only works with the RST corpus to access gold RST constituency trees. + file_split : str, one of {'corpus', 'dialogue', 'doc'} + Whether groups of files are generated for each corpus section, + dialogue (eg. for STAC) or document (eg. for RST-DT). + Returns ------- - mpack: Multipack + mpack : Multipack Multipack (= dict) from grouping to DataPack. """ + # WIP 2017-02-07 file_split + # support both doc- and corpus-centric calls + if file_split not in ('corpus', 'dialogue', 'doc'): + raise ValueError("file_split should be one of " + "\{'corpus', 'dialogue', 'doc'\}") + # end WIP file_split + + # setup result, load common files + mpack = dict() + # common files: vocabulary, labels vocab = load_vocab(vocab_file) - - with Torpor("Reading edus and pairings", quiet=not verbose): - edus, pairings = _process_edu_links(load_edus(edu_file), - load_pairings(pairings_file)) - - with Torpor("Reading features", quiet=not verbose): - labels = [UNKNOWN] + load_labels(feature_file) - # pylint: disable=unbalanced-tuple-unpacking - data, targets = load_svmlight_file(feature_file, - n_features=len(vocab)) - # pylint: enable=unbalanced-tuple-unpacking + labels = load_labels(labels_file) + assert labels[0] == UNKNOWN # WIP augment DataPack with the gold structure for each grouping if corpus_path is None: @@ -261,12 +433,118 @@ def load_multipack(edu_file, pairings_file, feature_file, vocab_file, # but this one and call slurp* with coarse_rels=False # end WIP + if file_split == 'corpus': + # one file per corpus split + edu_files = list(glob(edu_file)) + pairings_files = list(glob(pairings_file)) + feature_files = list(glob(feature_file)) + assert len(edu_files) == 1 + assert len(pairings_files) == 1 + assert len(feature_files) == 1 + edu_f = edu_files[0] + pairings_f = pairings_files[0] + feature_f = feature_files[0] + # load EDUs and pairings for each doc + doc_edus = defaultdict(list) + doc_pairings = dict() + with Torpor("Reading EDUs and pairings", quiet=not verbose): + edus, pairings = _process_edu_links(load_edus(edu_f), + load_pairings(pairings_f)) + # BEWARE _process_edu_links() prepends a unique fake root EDU + # to the list of EDUs ; it is currently shared between all + # groupings (docs), which makes me (MM) quite uncomfortable ; + # let's figure out a proper fix later, shall we? + grp2idc = groupings(pairings) # map group names to indices + for grp_name, pair_idc in grp2idc.items(): + # DIRTY initialize the list of EDUs for each grouping (doc) + # with the fake root EDU that is at (the global) edus[0] + doc_edus[grp_name].append(FAKE_ROOT) + # for RST-DT, grp_name is a doc_name ; for STAC it is a + # dialogue identifier + doc_pairings[grp_name] = [pairings[i] for i in pair_idc] + # distribute *real* EDUs to their grouping ; DIRTY again + for edu in edus[1:]: + doc_edus[edu.grouping].append(edu) + + # load the corresponding feature vectors and targets + doc_data = dict() + doc_targets = dict() + with Torpor("Reading features", quiet=not verbose): + data, targets = load_svmlight_file( + feature_f, n_features=len(vocab)) + for grp_name, pair_idc in grp2idc.items(): + doc_data[grp_name] = data[pair_idc] + doc_targets[grp_name] = targets[pair_idc] + # load the same info for CDUs + # 2017-02-07 inactive + doc_names = sorted(grp2idc.keys()) + doc_cdus, doc_cdu_pairings, doc_cdu_data, doc_cdu_targets = _load_multipack_cdus( + cdu_file, cdu_pairings_file, cdu_feature_file, vocab, doc_names, + verbose=verbose) + + elif file_split == 'doc': # TODO dialogue? + # one file per doc (2016-08-30) + edu_files = {os.path.basename(f).rsplit('.', 4)[0]: f + for f in glob(edu_file)} + pairings_files = {os.path.basename(f).rsplit('.', 4)[0]: f + for f in glob(pairings_file)} + feature_files = {os.path.basename(f).rsplit('.', 3)[0]: f + for f in glob(feature_file)} + + doc_names = sorted(edu_files.keys()) + # load EDUs and pairings for each doc + doc_edus = dict() + doc_pairings = dict() + with Torpor("Reading EDUs and pairings", quiet=not verbose): + for doc_name in doc_names: + edu_f = edu_files[doc_name] + pairings_f = pairings_files[doc_name] + edus, pairings = _process_edu_links(load_edus(edu_f), + load_pairings(pairings_f)) + # TODO for 'dialogue' too? + # each file should contain info from exactly one doc + grp_names = groupings(pairings).keys() + assert grp_names == [doc_name] + # store + doc_edus[doc_name] = edus + doc_pairings[doc_name] = pairings + # load the corresponding feature vectors and targets + doc_data = dict() + doc_targets = dict() + with Torpor("Reading features", quiet=not verbose): + for doc_name in doc_names: + feature_f = feature_files[doc_name] + # pylint: disable=unbalanced-tuple-unpacking + data, targets = load_svmlight_file(feature_f, + n_features=len(vocab)) + # pylint: enable=unbalanced-tuple-unpacking + doc_data[doc_name] = data + doc_targets[doc_name] = targets + + # load the same info for CDUs + # 2017-02-07 inactive + doc_cdus, doc_cdu_pairings, doc_cdu_data, doc_cdu_targets = _load_multipack_cdus( + cdu_file, cdu_pairings_file, cdu_feature_file, vocab, doc_names, + verbose=verbose) + + # build DataPack with Torpor("Build data packs", quiet=not verbose): - dpack = DataPack.load(edus, pairings, data, targets, ctargets, - labels, vocab) + for doc_name in doc_names: + dpack = DataPack.load( + doc_edus[doc_name], doc_pairings[doc_name], + doc_data[doc_name], doc_targets[doc_name], + # maybe we could avoid the dummy dict (that contains a + # unique pair) and just have an RSTTree here, but I am + # afraid it could cause inconsistencies with datapacks + # created with Datapack.vstack + dict([(doc_name, ctargets.get(doc_name, None))]), + # WIP CDU + doc_cdus[doc_name], doc_cdu_pairings[doc_name], + doc_cdu_data[doc_name], doc_cdu_targets[doc_name], + # end WIP CDU + labels, vocab) + mpack[doc_name] = dpack - mpack = {grp_name: dpack.selected(idxs) - for grp_name, idxs in groupings(pairings).items()} return mpack @@ -308,12 +586,21 @@ def mk_row(edu1, edu2): writer.writerow(mk_row(edu1, edu2)) -def load_predictions(edu_file): - """ - Read back predictions (see :doc:`../output`), returning a list - of triples: parent id, child id, relation label (or 'UNRELATED') +def load_predictions(edges_file): + """Read back predictions (see :doc:`../output`), returning a list + of triples: parent id, child id, relation label (or 'UNRELATED'). - :rtype: [(string, string, string)] + Parameters + ---------- + edges_file: str + Path to the file that contains predicted edges. + + Returns + ------- + edges_pred: list of (str, str, str) + List of predicted edges as triples (gov_id, dep_id, lbl_pred). + If (gov_id, dep_id) is predicted to be unattached, lbl_pred is + 'UNRELATED'. """ def mk_pair(row): 'interpret a single row' @@ -322,27 +609,29 @@ def mk_pair(row): oops = ('This row in the predictions file {efile} has {num} ' 'elements instead of the expected {expected}: ' '{row}') - raise IoException(oops.format(efile=edu_file, + raise IoException(oops.format(efile=edges_file, num=len(row), expected=expected_len, row=row)) return tuple(x.decode('utf-8') for x in row) - with open(edu_file, 'rb') as instream: + with open(edges_file, 'rb') as instream: reader = csv.reader(instream, dialect=csv.excel_tab) return [mk_pair(r) for r in reader if r] -def load_gold_predictions(pairings_file, feature_file, verbose=False): +def load_gold_predictions(pairings_file, feature_file, labels_file, + verbose=False): """ Load a pairings and feature file as though it were a set of predictions :rtype: [(string, string, string)] """ + labels = load_labels(labels_file) + pairings = load_pairings(pairings_file) with Torpor("Reading features", quiet=not verbose): - labels = load_labels(feature_file) # pylint: disable=unbalanced-tuple-unpacking _, targets = load_svmlight_file(feature_file) # pylint: enable=unbalanced-tuple-unpacking diff --git a/attelo/learning/local.py b/attelo/learning/local.py index 9258136..92394b3 100644 --- a/attelo/learning/local.py +++ b/attelo/learning/local.py @@ -2,10 +2,14 @@ Local classifiers """ +from __future__ import print_function + +import warnings + import numpy as np -from attelo.table import (DataPack, - for_labelling) +from attelo.cdu import CDU +from attelo.table import DataPack from .interface import (AttachClassifier, LabelClassifier) from .util import (relabel) @@ -14,9 +18,9 @@ class SklearnClassifier(object): - ''' - An scikit classifier used for any purpose - ''' + """ + An sklearn classifier used for any purpose + """ def __init__(self, learner): self._learner = learner pfunc = getattr(learner, "predict_proba", None) @@ -82,59 +86,123 @@ def important_features_multi(self, top_n): class SklearnAttachClassifier(AttachClassifier, SklearnClassifier): - ''' - A relatively simple way to get an attachment classifier: - just pass in a scikit classifier - ''' + """A relatively simple way to get an attachment classifier: + just pass in an sklearn classifier. - def __init__(self, learner): - """ - learner: scikit-compatible classifier - Use the given learner for label prediction. - """ + Parameters + ---------- + learner: sklearn API-compatible classifier + The learner to use for label prediction. + + pos_label: str or int, 1 by default + The class that codes an attachment decision. + """ + + def __init__(self, learner, pos_label=1): AttachClassifier.__init__(self) SklearnClassifier.__init__(self, learner) self._fitted = False + self.pos_label = pos_label - def fit(self, dpacks, targets, nonfixed_pairs=None): - # WIP select only the nonfixed pairs - if nonfixed_pairs is not None: - dpacks = [dpack.selected(nf_pairs) - for dpack, nf_pairs in zip(dpacks, nonfixed_pairs)] - targets = [target[nf_pairs] - for target, nf_pairs in zip(targets, nonfixed_pairs)] - + def fit(self, dpacks, targets): dpack = DataPack.vstack(dpacks) target = np.concatenate(targets) self._learner.fit(dpack.data, target) self._fitted = True return self - def predict_score(self, dpack, nonfixed_pairs=None): + def predict_score(self, dpack): if not self._fitted: raise ValueError('Fit not yet called') - # WIP pass only nonfixed pairs to the classifier - if nonfixed_pairs is not None: - dpack_filtd = dpack.selected(nonfixed_pairs) - else: - dpack_filtd = dpack - if self.can_predict_proba: - attach_idx = list(self._learner.classes_).index(1) - probs = self._learner.predict_proba(dpack_filtd.data) + # retro-compatibility + if not hasattr(self, 'pos_label'): + warnings.warn( + "Assuming pos_label=1 ; you are probably using an {0} " + "unpickled from an older version of educe.".format( + self.__class__.__name__), + UserWarning) + self.pos_label = 1 + # end retro-compatibility + attach_idx = list(self._learner.classes_).index(self.pos_label) + probs = self._learner.predict_proba(dpack.data) scores_pred = probs[:, attach_idx] else: - scores_pred = self._learner.decision_function(dpack_filtd.data) - - # WIP overwrite only the attachment scores of non-fixed pairs - if nonfixed_pairs is not None: - scores = np.copy(dpack.graph.attach) - scores[nonfixed_pairs] = scores_pred - else: - scores = scores_pred - - return scores + scores_pred = self._learner.decision_function(dpack.data) + if False: + # scoring of CDU pairings is currently de-activated + scores_pred = self.overwrite_scores_cdu(dpack, scores_pred) + return scores_pred + + def overwrite_scores_cdu(self, dpack, scores_pred): + """Overwrite scores of EDU pairings with scores of CDU pairings. + + Parameters + ---------- + dpack : DataPack + DataPack + + scores_pred : array of float, dimensions=(len(edu_pairings), 1) + Predicted scores for pairs of EDUs. + + Returns + ------- + scores_pred : array of float, dimensions=(len(edu_pairings), 1) + Updated array of predicted scores. + + Notes + ----- + All CDU-related code might be better off in a dedicated submodule + of parser. + """ + attach_idx = list(self._learner.classes_).index(self.pos_label) + # 2016-07-29 WIP compute scores for CDUs + # TODO find a cleaner way to compute these scores + epairs_map = {(src.id, tgt.id): i for i, (src, tgt) + in enumerate(dpack.pairings)} + # filter CDU pairs + cpairs_idc = [] # indices of selected CDU pairs + epairs_idc = [] # indices of corresponding EDU pairs + for i, (src, tgt) in enumerate(dpack.cdu_pairings): + esrc = src.members[0] if isinstance(src, CDU) else src.id + etgt = tgt.members[0] if isinstance(tgt, CDU) else tgt.id + if (esrc, etgt) in epairs_map: + cpairs_idc.append(i) + epairs_idc.append(epairs_map[(esrc, etgt)]) + # score pairs on CDUs, replace the score of the EDU pair if the + # score of the CDU pair is higher + if cpairs_idc: + print('woot!') # DEBUG + sel_cdu_data = dpack.cdu_data[cpairs_idc] + if self.can_predict_proba: + cdu_probs = self._learner.predict_proba(sel_cdu_data) + cdu_scores_pred = cdu_probs[:, attach_idx] + else: + cdu_scores_pred = self._learner.decision_function( + sel_cdu_data) + # DEBUG + if False: + epairs = [dpack.pairings[i] for i in epairs_idc] + epair_ids = [(src.id, tgt.id) for src, tgt in epairs] + print('epair scores') + for x, y in zip(epair_ids, + [scores_pred[i] for i in epairs_idc])[:30]: + print(x, y) + cpairs = [dpack.cdu_pairings[i] for i in cpairs_idc] + cpair_ids = [(src if isinstance(src, CDU) else src.id, + tgt if isinstance(tgt, CDU) else tgt.id) + for src, tgt in cpairs] + print('cpair scores') + for x, y in zip(cpair_ids, cdu_scores_pred)[:30]: + print(x, y) + raise ValueError('gne') + # end DEBUG + # was: np.maximum(scores_pred[epairs_idc], cdu_scores_pred) + scores_pred[epairs_idc] = cdu_scores_pred + # end WIP CDUs + + return scores_pred class SklearnLabelClassifier(LabelClassifier, SklearnClassifier): @@ -159,14 +227,7 @@ def __init__(self, learner): self._fitted = False self._labels = None # not yet learned - def fit(self, dpacks, targets, nonfixed_pairs=None): - # WIP select only the nonfixed pairs - if nonfixed_pairs is not None: - dpacks = [dpack.selected(nf_pairs) - for dpack, nf_pairs in zip(dpacks, nonfixed_pairs)] - targets = [target[nf_pairs] - for target, nf_pairs in zip(targets, nonfixed_pairs)] - + def fit(self, dpacks, targets): dpack = DataPack.vstack(dpacks) target = np.concatenate(targets) self._learner.fit(dpack.data, target) @@ -174,30 +235,15 @@ def fit(self, dpacks, targets, nonfixed_pairs=None): self._fitted = True return self - def predict_score(self, dpack, nonfixed_pairs=None): + def predict_score(self, dpack): if not self._fitted: raise ValueError('Fit not yet called') if self._labels is None: raise ValueError('No labels associated with this classifier') - dpack, _ = for_labelling(dpack, dpack.target) - - # WIP don't pass the fixed pairs to the classifier - if nonfixed_pairs is not None: - dpack_filtd = dpack.selected(nonfixed_pairs) - else: - dpack_filtd = dpack - # TODO non-probabilistic labellers - weights = self._learner.predict_proba(dpack_filtd.data) - lbl_scores_pred = relabel(self._labels, weights, dpack_filtd.labels) - - # WIP overwrite only the labelling scores of non-fixed pairs - if nonfixed_pairs is not None: - lbl_scores = np.copy(dpack.graph.label) - lbl_scores[nonfixed_pairs] = lbl_scores_pred - else: - lbl_scores = lbl_scores_pred + weights = self._learner.predict_proba(dpack.data) + lbl_scores_pred = relabel(self._labels, weights, dpack.labels) - return lbl_scores + return lbl_scores_pred diff --git a/attelo/learning/oracle.py b/attelo/learning/oracle.py index 6d0f0e5..0ce9c43 100644 --- a/attelo/learning/oracle.py +++ b/attelo/learning/oracle.py @@ -28,35 +28,24 @@ def __init__(self): super(AttachOracle, self).__init__() self.can_predict_proba = True - def fit(self, dpacks, targets, nonfixed_pairs=None): + def fit(self, dpacks, targets): return self - def predict_score(self, dpack, nonfixed_pairs=None): + def predict_score(self, dpack): """Predict 1.0 for gold attachments, 0.0 otherwise. Notes ----- - This assumes that gold attachments are coded as 1 ; this assumption - is currently baked in `attelo.table.for_attachment`. + This assumes that gold attachments are coded as 1 ; this + assumption is currently baked in `attelo.table.for_attachment`. TODO ---- [ ] rename and refactor to predict_proba(self, dpacks) """ - if nonfixed_pairs is not None: - y_true = dpack.target[nonfixed_pairs] - else: - y_true = dpack.target - + y_true = dpack.target score_true = np.where(y_true == 1, 1.0, 0.0) - - if nonfixed_pairs is not None: - res = np.copy(dpack.graph.attach) - res[nonfixed_pairs] = score_true - else: - res = score_true - - return res + return score_true class LabelOracle(LabelClassifier): @@ -75,10 +64,10 @@ def __init__(self): super(LabelOracle, self).__init__() self.can_predict_proba = True - def fit(self, dpacks, targets, nonfixed_pairs=None): + def fit(self, dpacks, targets): return self - def predict_score(self, dpack, nonfixed_pairs=None): + def predict_score(self, dpack): """Predict 1.0 for the gold label of edges. Non-gold edges are attributed "unknown" for their gold label. @@ -98,23 +87,14 @@ def predict_score(self, dpack, nonfixed_pairs=None): [ ] rename and refactor to predict_proba(self, dpacks) """ weights = dok_matrix((len(dpack), len(dpack.labels))) + + y_true = dpack.target + # for each pairing, if the true label is "unrelated", set 1.0 + # score to "unknown" instead (enables gold labelling on non-gold + # attachment) lbl_unrelated = dpack.label_number(UNRELATED) lbl_unk = dpack.label_number(UNKNOWN) - - if nonfixed_pairs is not None: - for i, lbl in enumerate(dpack.target): - if i in nonfixed_pairs: - if lbl == lbl_unrelated: - weights[i, lbl_unk] = 1.0 - else: - weights[i, lbl] = 1.0 - else: - weights[i, :] = dpack.graph.label[i] - else: - for i, lbl in enumerate(dpack.target): - if lbl == lbl_unrelated: - weights[i, lbl_unk] = 1.0 - else: - weights[i, lbl] = 1.0 + lbl_true = np.where(y_true != lbl_unrelated, y_true, lbl_unk) + weights[np.arange(len(weights)), lbl_true] = 1.0 return weights.todense() diff --git a/attelo/metrics/classification_structured.py b/attelo/metrics/classification_structured.py deleted file mode 100644 index e9d8b9b..0000000 --- a/attelo/metrics/classification_structured.py +++ /dev/null @@ -1,153 +0,0 @@ -"""Classification metrics for structured outputs. - -""" - -from collections import Counter -from itertools import chain, izip - -import numpy as np - - -def _unique_labels(y): - """Set of unique labels in y""" - return set(y_ij[1] for y_ij in - chain.from_iterable(y_i for y_i in y)) - - -def unique_labels(*ys): - """Extract an ordered array of unique labels. - - Parameters - ---------- - elt_type: string - Type of each element, determines how to find the label - - See also - -------- - This is the structured version of - `sklearn.utils.multiclass.unique_labels` - """ - ys_labels = set(chain.from_iterable(_unique_labels(y) for y in ys)) - # TODO check the set of labels contains a unique (e.g. string) type - # of values - return np.array(sorted(ys_labels)) - - -def precision_recall_fscore_support(y_true, y_pred, labels=None, - average=None): - """Compute precision, recall, F-measure and support for each class. - - The support is the number of occurrences of each class in - ``y_true``. - - This is essentially a structured version of - sklearn.metrics.classification.precision_recall_fscore_support . - - It should apply equally well to lists of constituency tree spans - and lists of dependency edges. - - Parameters - ---------- - y_true: list of iterable - Ground truth target structures, encoded in a sparse format (e.g. - list of edges or span descriptions). - - y_pred: list of iterable - Estimated target structures, encoded in a sparse format (e.g. list - of edges or span descriptions). - - labels: list, optional - The set of labels to include, and their order if ``average is - None``. - - average: string, [None (default), 'binary', 'micro', 'macro'] - If ``None``, the scores for each class are returned. Otherwise, - this determines the type of averaging performed on the data: - - ``'binary'``: - Only report results for the positive class. - This is applicable only if targets are binary. - ``'micro'``: - Calculate metrics globally by counting the total true - positives, false negatives and false positives. - ``'macro'``: - Calculate metrics for each label, and find their unweighted - mean. This does not take label imbalance into account. - - Returns - ------- - precision: float (if average is not None) or array of float, shape=\ - [n_unique_labels] - - recall: float (if average is not None) or array of float, shape=\ - [n_unique_labels] - - fscore: float (if average is not None) or array of float, shape=\ - [n_unique_labels] - - support: int (if average is not None) or array of int, shape=\ - [n_unique_labels] - The number of occurrences of each label in ``ctree_true``. - """ - average_options = frozenset([None, 'micro', 'macro']) - if average not in average_options: - raise ValueError('average has to be one of' + - str(average_options)) - # TMP - if average == 'macro': - raise NotImplementedError('average currently has to be micro or None') - # end TMP - - # gather an ordered list of unique labels from y_true and y_pred - present_labels = unique_labels(y_true, y_pred) - - if labels is None: - labels = present_labels - # n_labels = None - else: - # EXPERIMENTAL - labels = [lbl for lbl in labels if lbl in present_labels] - # n_labels = len(labels) - # FIXME complete/fix this - # raise ValueError('Parameter `labels` is currently unsupported') - # end EXPERIMENTAL - - # compute tp_sum, pred_sum, true_sum - # true positives for each tree - tp = [set(yi_true) & set(yi_pred) - for yi_true, yi_pred in izip(y_true, y_pred)] - - # TODO find a nicer and faster design that resembles sklearn's, e.g. - # use np.bincount instead of collections.Counter - tp_sum = Counter(y_ij[1] for y_ij in chain.from_iterable(tp)) - true_sum = Counter(y_ij[1] for y_ij in chain.from_iterable(y_true)) - pred_sum = Counter(y_ij[1] for y_ij in chain.from_iterable(y_pred)) - # transform to np arrays of floats - tp_sum = np.array([float(tp_sum[lbl]) for lbl in labels]) - true_sum = np.array([float(true_sum[lbl]) for lbl in labels]) - pred_sum = np.array([float(pred_sum[lbl]) for lbl in labels]) - - # TODO rewrite to compute by summing over scores broken down by label - if average == 'micro': - tp_sum = np.array([tp_sum.sum()]) - true_sum = np.array([true_sum.sum()]) - pred_sum = np.array([pred_sum.sum()]) - - # finally compute the desired statistics - # when the div denominator is 0, assign 0.0 (instead of np.inf) - precision = tp_sum / pred_sum - precision[pred_sum == 0] = 0.0 - - recall = tp_sum / true_sum - recall[true_sum == 0] = 0.0 - - f_score = 2 * (precision * recall) / (precision + recall) - f_score[precision + recall == 0] = 0.0 - - if average is not None: - precision = np.average(precision) - recall = np.average(recall) - f_score = np.average(f_score) - true_sum = np.average(true_sum) # != sklearn: we keep the support - - return precision, recall, f_score, true_sum diff --git a/attelo/metrics/constituency.py b/attelo/metrics/constituency.py deleted file mode 100644 index 06040c3..0000000 --- a/attelo/metrics/constituency.py +++ /dev/null @@ -1,314 +0,0 @@ -"""Metrics for constituency trees. - -TODO ----- -* [ ] factor out the report from the parseval function, see -`sklearn.metrics.classification.classification_report` -* [ ] refactor the selection functions that enable to break down -evaluations, to avoid almost duplicates (as currently) -""" - -from __future__ import print_function - -import numpy as np - -from .classification_structured import (precision_recall_fscore_support, - unique_labels) -from .util import get_spans - - -# label extraction functions -LBL_FNS = [ - ('S', lambda span: 1), - ('S+N', lambda span: span[1]), - ('S+R', lambda span: span[2]), - ('S+N+R', lambda span: '{}-{}'.format(span[2], span[1])), -] - - -# PARSEVAL metrics adapted to the evaluation of discourse parsers, -# with options to get meaningful variants in specific settings -def discourse_parseval_scores(ctree_true, ctree_pred, - labels=None, average=None): - """Compute discourse PARSEVAL scores for ctree_pred wrt ctree_true. - - Parameters - ---------- - ctree_true : list of list of RSTTree or SimpleRstTree - - ctree_pred : list of list of RSTTree or SimpleRstTree - - labels : list of string, optional - Corresponds to sklearn's target_names IMO - - Returns - ------- - precision : float (if average is not None) or array of float, shape =\ - [n_unique_labels] - Weighted average of the precision of each class. - - recall : float (if average is not None) or array of float, shape =\ - [n_unique_labels] - - fbeta_score : float (if average is not None) or array of float, shape =\ - [n_unique_labels] - - support : int (if average is not None) or array of int, shape =\ - [n_unique_labels] - The number of occurrences of each label in ``ctree_true``. - - References - ---------- - .. [1] `Daniel Marcu (2000). "The theory and practice of discourse - parsing and summarization." MIT press. - - """ - - # extract descriptions of spans from the true and pred trees - spans_true = [get_spans(ct_true) for ct_true in ctree_true] - spans_pred = [get_spans(ct_pred) for ct_pred in ctree_pred] - # use lbl_fn to define labels - spans_true = [[(span[0], lbl_fn(span)) for span in spans] - for spans in spans_true] - spans_pred = [[(span[0], lbl_fn(span)) for span in spans] - for spans in spans_pred] - - p, r, f, s = precision_recall_fscore_support(spans_true, spans_pred, - labels=labels, - average=average) - return p, r, f, s - - -def parseval_report(ctree_true, ctree_pred, metric_types=None, digits=4, - stringent=False): - """Build a text report showing the PARSEVAL discourse metrics. - - This is the simplest report we need to generate, it corresponds - to the arrays of results from the literature. - Metrics are calculated globally (average='micro'). - - Parameters - ---------- - metric_types: list of strings, optional - Metrics that need to be included in the report ; if None is - given, defaults to ['S', 'S+N', 'S+R', 'S+N+R']. - """ - if metric_types is None: - metric_types = ['S', 'S+N', 'S+R', 'S+N+R'] - if set(metric_types) - set(x[0] for x in LBL_FNS): - raise ValueError('Unknown metric types in {}'.format(metric_types)) - - # FIXME refactor in tandem with discourse_parseval_scores, to - # get a coherent and non-redundant API - # extract descriptions of spans from the true and pred trees - spans_true = [get_spans(ct_true) for ct_true in ctree_true] - spans_pred = [get_spans(ct_pred) for ct_pred in ctree_pred] - - # prepare report - width = max(len(str(x)) for x in metric_types) - width = max(width, digits) - - headers = ["precision", "recall", "f1-score", "support"] - fmt = '%% %ds' % width # first col: class name - fmt += ' ' - fmt += ' '.join(['% 9s' for _ in headers]) - fmt += '\n' - - headers = [""] + headers - report = fmt % tuple(headers) - report += '\n' - # end prepare report - - metric2lbl_fn = dict(LBL_FNS) - - for metric_type in metric_types: - lbl_fn = metric2lbl_fn[metric_type] - # possibly filter data - sp_true = spans_true - sp_pred = spans_pred - if stringent: - # stringent variant: - if metric_type == 'S': - # * S: exclude leaves - sp_true = [[xi for xi in x if xi[0][1] != xi[0][0]] - for x in sp_true] - sp_pred = [[xi for xi in x if xi[0][1] != xi[0][0]] - for x in sp_pred] - elif False and metric_type in ['S+R', 'S+N+R']: - # * S+R, S+N+R: exclude 'span' - sp_true = [[xi for xi in x if xi[2] != 'span'] - for x in sp_true] - sp_pred = [[xi for xi in x if xi[2] != 'span'] - for x in sp_pred] - # end filter - y_true = [[(span[0], lbl_fn(span)) for span in spans] - for spans in sp_true] - y_pred = [[(span[0], lbl_fn(span)) for span in spans] - for spans in sp_pred] - # calculate metric - p, r, f1, s = precision_recall_fscore_support(y_true, y_pred, - average='micro') - # report - values = [metric_type] - for v in (p, r, f1): - values += ["{0:0.{1}f}".format(v, digits)] - values += ["{0}".format(s)] - report += fmt % tuple(values) - return report - - -def parseval_detailed_report(ctree_true, ctree_pred, - metric_type='S+R', - labels=None, - average=None, - sort_by_support=True, - digits=4): - """Build a text report showing the PARSEVAL discourse metrics. - - FIXME model after sklearn.metrics.classification.classification_report - - Parameters - ---------- - ctree_true : list of RSTTree or SimpleRstTree - Ground truth (correct) target structures. - - ctree_pred : list of RSTTree or SimpleRstTree - Estimated target structures as predicted by a parser. - - labels : list of string, optional - Relation labels to include in the evaluation. - FIXME Corresponds more to target_names in sklearn IMHO. - - lbl_fn : function from tuple((int, int), string, string) to string - Label extraction function - - digits : int - Number of digits for formatting output floating point values. - - Returns - ------- - report : string - Text summary of the precision, recall, F1 score, support for each - class (or micro-averaged over all classes). - - References - ---------- - .. [1] `Daniel Marcu (2000). "The theory and practice of discourse - parsing and summarization." MIT press. - - """ - metric2lbl_fn = dict(LBL_FNS) - lbl_fn = metric2lbl_fn[metric_type] - - # extract descriptions of spans from the true and pred trees - spans_true = [get_spans(ct_true) for ct_true in ctree_true] - spans_pred = [get_spans(ct_pred) for ct_pred in ctree_pred] - # use lbl_fn to extract the label of interest - y_true = [[(span[0], lbl_fn(span)) for span in spans] - for spans in spans_true] - y_pred = [[(span[0], lbl_fn(span)) for span in spans] - for spans in spans_pred] - - present_labels = unique_labels(y_true, y_pred) - - if labels is None: - labels = present_labels - n_labels = None - else: - # currently not tested - n_labels = len(labels) - labels = np.hstack([labels, np.setdiff1d(present_labels, labels, - assume_unique=True)]) - - last_line_heading = 'avg / total' - - width = max(len(str(lbl)) for lbl in labels) - width = max(width, len(last_line_heading), digits) - - headers = ["precision", "recall", "f1-score", "support"] - fmt = '%% %ds' % width # first col: class name - fmt += ' ' - fmt += ' '.join(['% 9s' for _ in headers]) - fmt += '\n' - - headers = [""] + headers - report = fmt % tuple(headers) - report += '\n' - - # call with average=None to compute per-class scores, then - # compute average here and print it - p, r, f1, s = precision_recall_fscore_support(y_true, y_pred, - labels=labels, - average=average) - sorted_ilbls = enumerate(labels) - if sort_by_support: - sorted_ilbls = sorted(sorted_ilbls, key=lambda x: s[x[0]], - reverse=True) - # one line per label - for i, label in sorted_ilbls: - values = [label] - for v in (p[i], r[i], f1[i]): - values += ["{0:0.{1}f}".format(v, digits)] - values += ["{0}".format(s[i])] - if average is None: # print per-class scores for average=None only - report += fmt % tuple(values) - - # print only if per-class scores - if average is None: - report += '\n' - - # compute averages for the bottom line - values = [last_line_heading] - for v in (np.average(p, weights=s), - np.average(r, weights=s), - np.average(f1, weights=s)): - values += ["{0:0.{1}f}".format(v, digits)] - values += ['{0}'.format(np.sum(s))] - report += fmt % tuple(values) - - return report - - -def parseval_reports(ctree_true, ctree_pred, labels=None, average=None, - digits=2): - """Build a text report showing the PARSEVAL discourse metrics. - - FIXME model after sklearn.metrics.classification.classification_report - - Parameters - ---------- - ctree_true : list of RSTTree or SimpleRstTree - Ground truth (correct) target structures. - - ctree_pred : list of RSTTree or SimpleRstTree - Estimated target structures as predicted by a parser. - - labels : list of string, optional - Relation labels to include in the evaluation. - FIXME Corresponds more to target_names in sklearn IMHO. - - digits : int - Number of digits for formatting output floating point values. - - Returns - ------- - report : string - Text summary of the precision, recall, F1 score, support for each - class (or micro-averaged over all classes). - - References - ---------- - .. [1] `Daniel Marcu (2000). "The theory and practice of discourse - parsing and summarization." MIT press. - - """ - # extract one report per type of metric - reports = [] - for metric_type, lbl_fn in LBL_FNS: - lbls = labels if metric_type in ['S+R', 'S+N+R'] else None - reports.append((metric_type, - parseval_report(ctree_true, ctree_pred, lbl_fn, - labels=lbls, - average=average, - digits=digits))) - return reports diff --git a/attelo/metrics/deptree.py b/attelo/metrics/deptree.py index 4c8bfa2..a912efe 100644 --- a/attelo/metrics/deptree.py +++ b/attelo/metrics/deptree.py @@ -1,12 +1,18 @@ """Common metrics on dependency trees. + +As of 2017-05-18, all implementations assume that _true and _pred both +rely on the same segmentation. """ +from __future__ import absolute_import, print_function + +from collections import Counter import itertools import numpy as np -def compute_uas_las(dtree_true, dtree_pred): +def compute_uas_las(dtree_true, dtree_pred, metrics=None, doc_names=None): """Compute dependency metrics for trees in dtree_pred wrt dtree_true. The computed metrics are the traditional UAS and LAS, plus LS @@ -14,48 +20,110 @@ def compute_uas_las(dtree_true, dtree_pred): Parameters ---------- - dtree_true: list of RstDepTree + dtree_true : list of RstDepTree Reference trees - dtree_pred: list of RstDepTree + dtree_pred : list of RstDepTree Predicted trees + metrics : list of str + If None, defaults to ['U', 'R'] aka. UAS and LAS. Possible + values in {'U', 'R', 'R+N', 'R+O', 'O+N', 'F', 'tag_R'}. + Returns ------- - (uas, las, ls): (float, float, float) - The Unlabelled and Labelled Attachment Scores, plus the - Labelling Score (new). + res : tuple of float + Score for each metric in order. """ - nb_ua_ok = 0 # correct unlabelled deps - nb_la_ok = 0 # correct labelled deps - nb_l_ok = 0 # correct labellings (right labels, possibly wrong heads) + # 'U': correct unlabelled deps ; was: nb_ua_ok + # 'R': correct labelled deps ; was: nb_la_ok + # 'tag_R': correct labellings: right labels, possibly wrong heads ; was: nb_l_ok + # 'R+N': relation and nuclearity + # 'R+O': relation and order + # 'F': relation, order, nuclearity + nb_tp = Counter({k: 0 for k in metrics}) nb_tot = 0 # total deps - for dt_true, dt_pred in itertools.izip(dtree_true, dtree_pred): - # heads and labels are stored as two lists + for i, (dt_true, dt_pred) in enumerate( + itertools.izip(dtree_true, dtree_pred)): + if doc_names is not None: + doc_name = doc_names[i] # for verbose/debug + tp = dict() + tp_bins = dict() # exclude fake root from metrics - heads_true = dt_true.heads[1:] - labels_true = dt_true.labels[1:] - - heads_pred = dt_pred.heads[1:] - labels_pred = dt_pred.labels[1:] - - for i in range(len(heads_pred)): - # attachments - if heads_pred[i] == heads_true[i]: - nb_ua_ok += 1 - if labels_pred[i] == labels_true[i]: - nb_la_ok += 1 - # NEW evaluate labelling only - if labels_pred[i] == labels_true[i]: - nb_l_ok += 1 - nb_tot += 1 - - score_uas = float(nb_ua_ok) / nb_tot - score_las = float(nb_la_ok) / nb_tot - score_ls = float(nb_l_ok) / nb_tot # NEW - - return (score_uas, score_las, score_ls) + # head : dependencies + if any(x in set(['U', 'N', 'R', 'O', 'R+N', 'R+O', 'O+N', 'F']) + for x in metrics): + heads_true = np.array(dt_true.heads[1:]) + heads_pred = np.array(dt_pred.heads[1:]) + tp['U'] = heads_true == heads_pred + tp_bins['U'] = heads_true[tp['U']] + # relation tag + if any(x in set(['tag_R', 'R', 'R+N', 'R+O', 'F']) for x in metrics): + labels_true = np.array(dt_true.labels[1:]) + labels_pred = np.array(dt_pred.labels[1:]) + tp['tag_R'] = labels_true == labels_pred + tp_bins['tag_R'] = labels_true[tp['tag_R']] + # dep + tp['R'] = np.logical_and(tp['U'], tp['tag_R']) + tp_bins['R'] = labels_true[tp['R']] + # nuclearity tag + if any(x in set(['tag_N', 'N', 'R+N', 'O+N', 'F']) for x in metrics): + nucs_true = np.array(dt_true.nucs[1:]) + nucs_pred = np.array(dt_pred.nucs[1:]) + tp['tag_N'] = nucs_true == nucs_pred + tp_bins['tag_N'] = nucs_true[tp['tag_N']] + # dep + tp['N'] = np.logical_and(tp['U'], tp['tag_N']) + tp_bins['N'] = nucs_true[tp['N']] + # order tag + if any(x in set(['tag_O', 'O', 'R+O', 'O+N', 'F']) for x in metrics): + rnks_true = np.array(dt_true.ranks[1:]) + rnks_pred = np.array(dt_pred.ranks[1:]) + tp['tag_O'] = rnks_true == rnks_pred + tp_bins['tag_O'] = rnks_true[tp['tag_O']] + # dep + tp['O'] = np.logical_and(tp['U'], tp['tag_O']) + tp_bins['O'] = rnks_true[tp['O']] + + # dep on complex labels, build on simpler labelled deps + if 'R+O' in metrics: + tp['R+O'] = np.logical_and(tp['R'], tp['O']) + tp_bins['R+O'] = np.array([ + (x, y) for x, y in zip( + labels_true[tp['R+O']], rnks_true[tp['R+O']]) + ]) + if 'R+N' in metrics: + tp['R+N'] = np.logical_and(tp['R'], tp['N']) + tp_bins['R+N'] = np.array([ + (x, y) for x, y in zip( + labels_true[tp['R+N']], nucs_true[tp['R+N']]) + ]) + if 'O+N' in metrics: + tp['O+N'] = np.logical_and(tp['N'], tp['O']) + tp_bins['O+N'] = np.array([ + (x, y) for x, y in zip( + labels_true[tp['O+N']], rnks_true[tp['O+N']]) + ]) + # full + if 'F' in metrics: + tp['F'] = np.logical_and.reduce((tp['R'], tp['N'], tp['O'])) + tp_bins['F'] = np.array([ + (x, y, z) for x, y, z in zip( + labels_true[tp['F']], nucs_true[tp['F']], + rnks_true[tp['F']]) + ]) + + # for each metric, update the number of true positives with + # the count for the current instance + for k, v in nb_tp.items(): + nb_tp[k] = v + len(tp_bins[k]) + + nb_tot += len(heads_true) + + scores = {k: float(nb_tp[k]) / nb_tot for k in nb_tp} + res = tuple([scores[k] for k in metrics]) + return res def compute_uas_las_listcomp(dtree_true, dtree_pred): @@ -166,3 +234,270 @@ def compute_uas_las_np(dtree_true, dtree_pred): score_ls = float(ls_num) / nb_tot # NEW return (score_uas, score_las, score_ls) + + +# 2016-09-30 undirected variants +def compute_uas_las_undirected(dtree_true, dtree_pred): + """Compute dependency metrics for trees in dtree_pred wrt dtree_true. + + The computed metrics are the traditional UAS and LAS, plus LS + for Labelling Score (counts of correct labels, regardless of head). + + Parameters + ---------- + dtree_true: list of RstDepTree + Reference trees + + dtree_pred: list of RstDepTree + Predicted trees + + Returns + ------- + (uas, las, ls): (float, float, float) + The Unlabelled and Labelled Attachment Scores, plus the + Labelling Score (new). + """ + nb_ua_ok = 0 # correct unlabelled deps + nb_la_ok = 0 # correct labelled deps + nb_tot = 0 # total deps + + for dt_true, dt_pred in itertools.izip(dtree_true, dtree_pred): + # undirected dependencies are equivalent to the span they cover + # each span is a tuple with a tuple inside ((fst, snd), lbl) + spans_true = set((tuple(sorted((gov, dep))), lbl) + for dep, (gov, lbl) + in enumerate(zip(dt_true.heads[1:], dt_true.labels[1:]), + start=1)) + spans_pred = set((tuple(sorted((gov, dep))), lbl) + for dep, (gov, lbl) + in enumerate(zip(dt_pred.heads[1:], dt_pred.labels[1:]), + start=1)) + nb_tot += len(spans_pred) + nb_ua_ok += len(set(x[0] for x in spans_true).intersection( + set(x[0] for x in spans_pred))) + nb_la_ok += len(spans_true.intersection(spans_pred)) + + score_uas = float(nb_ua_ok) / nb_tot + score_las = float(nb_la_ok) / nb_tot + + return (score_uas, score_las) + + +def dep_compact_report(parser_true, d_preds, dep_metrics, doc_names, + labelset_true, digits=3, percent=False, + out_format='text'): + """Compact textual report of parser accuracies with dependency metrics. + + Parameters + ---------- + parser_true : str + Name of the parser used as reference. + d_preds : list of (str, dict from str to RstDepTree) + List of predicted head-ordered d-trees for each parser. + dep_metrics : list of str + List of dependency metrics to include in the report. + doc_names : list of str? + TODO + labelset_true : ? + TODO + digits : int, defaults to 3 + Significant digits for rounding. + percent : boolean, defaults to False + Display scores as percentages. + out_format : one of {'text', 'latex'} + Output format. + + Returns + ------- + report : str + Textual report + """ + out_format_options = ('text', 'latex') + if out_format not in out_format_options: + raise ValueError('out_format has to be one of ' + + str(out_format_options)) + + # report + # * table format + headers = dep_metrics + headers = ["parser"] + headers + if out_format == 'latex': + # bold font for column headers + headers = ['\\textbf{{{}}}'.format(x) for x in headers] + # width of first column (parser name) + width = max([len(parser_name) for parser_name, _ in d_preds] + + [len(headers[0])]) + fmt = '%% %ds' % width # first col: parser name + if out_format == 'latex': + fmt += ' &' + fmt += ' &'.join(['% {}s'.format(len(x)) for x in headers[1:]]) + fmt += ' \\\\' # print "\\" + else: # if out_format == 'text': + fmt += ' ' + fmt += ' '.join(['% 9s' for _ in headers[1:]]) + fmt += '\n' + + report = "" + if out_format == 'latex': + report += '\n'.join([ + '\\begin{table}[h]', + '\\caption{\\label{dtree-eval} Dependency evaluation. U = unlabelled dependencies, O = dependencies labelled with the order of attachment, N = dependencies labelled with the nuclearity alone, O+N = order and nuclearity, R = relation, R+N = relation and nuclearity, F = fully labelled dependencies.}', + '\\begin{center}', + '\\begin{tabular}{' + 'l' * len(headers) +'}', + '\\toprule', + '' + ]) + report += fmt % tuple(headers) + if out_format == 'latex': + report += '\\midrule\n' + else: + report += '\n' + + # display percentages + dep_digits = digits - 2 if percent else digits + # end table format and header line + + # * table content + # dtree_true_list = [dtree_true[doc_name] for doc_name in doc_names] + # FIXME + dtree_true_list = [] + for parser_name, dtree_pred in d_preds: + if parser_name == parser_true: + dtree_true_list = [dtree_pred[doc_name] for doc_name in doc_names] + break + # end FIXME + # _pred + for parser_name, dtree_pred in d_preds: + dtree_pred_list = [dtree_pred[doc_name] for doc_name in doc_names] + # check that labelset_pred is a subset of labelset_true + labelset_pred = set(itertools.chain.from_iterable( + x.labels for x in dtree_pred_list)) + try: + assert labelset_pred.issubset(labelset_true) + except AssertionError: + print(parser_name) + print('T & P', sorted(labelset_true.intersection(labelset_pred))) + print('T - P', sorted(labelset_true - labelset_pred)) + print('P - T', sorted(labelset_pred - labelset_true)) + raise + # end check + all_scores = [] + all_scores += list(compute_uas_las( + dtree_true_list, dtree_pred_list, metrics=dep_metrics, + doc_names=doc_names)) + # append to report + values = ['{pname: <{fill}}'.format(pname=parser_name, fill=width)] + for v in all_scores: + if percent: + v = v * 100.0 + values += ["{0:0.{1}f}".format(v, dep_digits)] + report += fmt % tuple(values) + # LaTeX footer if relevant + if out_format == 'latex': + report += '\n'.join([ + '\\bottomrule', + '\\end{tabular}', + '\\end{center}', + '\\end{table}' + ]) + # end table content + + # replace underscores in parser names etc + report = report.replace('_', ' ') + return report + + +def dep_similarity(d_preds, doc_names, labelset_true, dep_metric=None, + digits=3, percent=False, out_format='text'): + """Compact textual report of parser accuracies with dependency metrics. + + Parameters + ---------- + d_preds : list of (str, dict from str to RstDepTree) + List of predicted head-ordered d-trees for each parser. + doc_names : list of str + List of document names. + labelset_true : list of str + List of true labels. + dep_metric : str, optional + Dependency metric to use ; defaults to 'U' (aka UAS). + digits : int, defaults to 3 + Significant digits for rounding. + percent : boolean, defaults to False + Display scores as percentages. + out_format : one of {'text', 'latex'} + Output format. + + Returns + ------- + report : str + Textual report + """ + out_format_options = ('text', 'latex') + if out_format not in out_format_options: + raise ValueError('out_format has to be one of ' + + str(out_format_options)) + + if dep_metric is None: + dep_metric = 'U' + + # prepare scaffold for report + width = max(len(parser_name) for parser_name, _ in d_preds) + headers = [k[:7] for k, v in d_preds] + # if we wanted to print the support, would be here for col name + fmt = '%% %ds' % width # first col: parser name + if out_format == 'latex': + fmt += ' &' + fmt += '&'.join(['% 9s' for _ in headers]) + fmt += '\\\\' # print "\\" + else: # if out_format == 'text': + fmt += ' ' + fmt += ' '.join(['% 9s' for _ in headers]) + fmt += '\n' + headers = [""] + headers + + report = "" + if out_format == 'latex': + report += '\n'.join([ + '\\begin{table}[h]', + '\\caption{\\label{dtree-sim} Pairwise similarity between parsers predictions, dependency metric U.}', + '\\begin{center}', + '\\begin{tabular}{' + 'l' * len(headers) +'}', + '\\toprule', + '' + ]) + report += fmt % tuple(headers) + report += '\n' + if out_format == 'latex': + report += '\\midrule\n' + + # display percentages + if percent: + digits = digits - 2 + + # use each parser as reference, in turn + for parser_true, dtree_true in d_preds: + values = [parser_true] # name of row + # get list of dtrees + dtree_true_list = [dtree_true[doc_name] for doc_name in doc_names] + for parser_name, dtree_pred in d_preds: + dtree_pred_list = [dtree_pred[doc_name] for doc_name in doc_names] + # compute score + f1 = compute_uas_las( + dtree_true_list, dtree_pred_list, metrics=[dep_metric], + doc_names=doc_names)[0] + # fill report + values += ["{0:0.{1}f}".format(f1 * 100.0 if percent else f1, + digits)] + report += fmt % tuple(values) + + if out_format == 'latex': + report += '\n'.join([ + '\\bottomrule', + '\\end{tabular}', + '\\end{center}', + '\\end{table}' + ]) + report = report.replace('_', ' ') + + return report diff --git a/attelo/metrics/util.py b/attelo/metrics/util.py index fee8aa2..53b9397 100644 --- a/attelo/metrics/util.py +++ b/attelo/metrics/util.py @@ -18,6 +18,7 @@ from educe.rst_dt.annotation import (EDU as EduceEDU, RSTTree, SimpleRSTTree) +from educe.rst_dt.corpus import mk_key from educe.rst_dt.dep2con import (deptree_to_simple_rst_tree, DummyNuclearityClassifier, InsideOutAttachmentRanker) @@ -27,19 +28,13 @@ from attelo.table import UNKNOWN -def get_oracle_ctrees(dep_edges, att_edus, - nuc_strategy="unamb_else_most_frequent", - rank_strategy="closest-intra-rl-inter-rl", - prioritize_same_unit=True, - strict=False): - """Build the oracle constituency tree(s) for a dependency tree. +def barebones_rst_deptree(dep_edges, att_edus, strict=False): + """Get a barebones RstDepTree: only heads and labels. Parameters ---------- - dep_edges: dict(string, [(string, string, string)]) - Edges for each document, indexed by doc name - Cf. type of return value from - irit-rst-dt/ctree.py:load_attelo_output_file() + dep_edges: [(string, string, string)] + List of edges for the document (gov_id, dep_id, lbl). att_edus: cf return type of attelo.io.load_edus EDUs as they are known to attelo strict: boolean, True by default @@ -48,8 +43,10 @@ def get_oracle_ctrees(dep_edges, att_edus, Returns ------- - ctrees: list of RstTree - There can be several e.g. for leaky sentences. + dtree: RstDepTree + Barebones dependency tree. + edu2sent: dict(str, dict(int, int)) + For each doc_name, map EDU number to sentence index. """ # rebuild educe EDUs from their attelo description # and group them by doc_name @@ -82,24 +79,16 @@ def get_oracle_ctrees(dep_edges, att_edus, assert len(educe_edus) == 1 # then restrict to this document doc_name = educe_edus.keys()[0] - educe_edus = educe_edus[doc_name] + doc_edus = educe_edus[doc_name] edu2sent_idx = edu2sent_idx[doc_name] # sort EDUs by num - educe_edus = list(sorted(educe_edus, key=lambda e: e.num)) + doc_edus = list(sorted(doc_edus, key=lambda e: e.num)) # rebuild educe-style edu2sent ; prepend 0 for the fake root - edu2sent = [0] + [edu2sent_idx[e.num] for e in educe_edus] - # classifiers for nuclearity and ranking - # FIXME declare, fit and predict upstream... - # nuclearity - nuc_classifier = DummyNuclearityClassifier(strategy=nuc_strategy) - nuc_classifier.fit([], []) # empty X and y for dummy fit - # ranking classifier - rank_classifier = InsideOutAttachmentRanker( - strategy=rank_strategy, - prioritize_same_unit=prioritize_same_unit) - + edu2sent = [0] + [edu2sent_idx[e.num] for e in doc_edus] + # 2017-02-10 create origin (FileId) + origin = mk_key(doc_name) # rebuild RstDepTrees - dtree = RstDepTree(educe_edus) + dtree = RstDepTree(edus=doc_edus, origin=origin) for src_id, tgt_id, lbl in dep_edges: if src_id == 'ROOT': if lbl not in ['ROOT', UNKNOWN]: @@ -112,53 +101,82 @@ def get_oracle_ctrees(dep_edges, att_edus, dtree.set_root(gid2num[tgt_id]) else: dtree.add_dependency(gid2num[src_id], gid2num[tgt_id], lbl) - # add nuclearity: heuristic baseline + return dtree, edu2sent + + +def get_oracle_ctrees(dep_edges, att_edus, + nuc_strategy="unamb_else_most_frequent", + rank_strategy="sdist-edist-rl", + prioritize_same_unit=True, + strict=False, + allow_forest=False): + """Build the oracle constituency tree(s) for a dependency tree. + + Parameters + ---------- + dep_edges : dict(string, [(string, string, string)]) + Edges for each document, indexed by doc name + Cf. type of return value from + irit-rst-dt/ctree.py:load_attelo_output_file() + + att_edus : cf return type of attelo.io.load_edus + EDUs as they are known to attelo + + strict : boolean, True by default + If True, any link from ROOT to an EDU that is neither 'ROOT' nor + UNRELATED raises an exception, otherwise a warning is issued. + + allow_forest : boolean, False by default + If True, allows several real roots in the d-tree, hence a forest + of RST c-trees. + + Returns + ------- + ctrees: list of RstTree + There can be several e.g. for leaky sentences. + """ + # get a barebones RstDepTree + dtree, edu2sent = barebones_rst_deptree(dep_edges, att_edus, + strict=strict) + # flesh out by adding nuclearity and ranking, from heuristic + # (pseudo-)classifiers + # FIXME declare, fit and predict upstream... + # * nuclearity + nuc_classifier = DummyNuclearityClassifier(strategy=nuc_strategy) + nuc_classifier.fit([], []) # empty X and y for dummy fit dtree.nucs = nuc_classifier.predict([dtree])[0] + # * rank + rank_classifier = InsideOutAttachmentRanker( + strategy=rank_strategy, + prioritize_same_unit=prioritize_same_unit) + rank_classifier.fit([], []) # add rank: some strategies require a mapping from EDU to sentence - # EXPERIMENTAL attach array of sentence index for each EDU in tree - dtree.sent_idx = edu2sent - # end EXPERIMENTAL + # WIP attach array of sentence index for each EDU in tree + dtree.sent_idx = edu2sent # FIXME dtree.ranks = rank_classifier.predict([dtree])[0] # end NEW - # create pred ctree try: - bin_srtrees = deptree_to_simple_rst_tree(dtree, allow_forest=True) - if False: # EXPERIMENTAL - # currently False to run on output that already has - # labels embedding nuclearity - bin_srtrees = [SimpleRSTTree.incorporate_nuclearity_into_label( - bin_srtree) for bin_srtree in bin_srtrees] + # FIXME replace with a straight call to deptree_to_rst_tree + bin_srtrees = deptree_to_simple_rst_tree( + dtree, allow_forest=allow_forest) + # FIXME inconsistent API + if not allow_forest: + # if allow_forest is False, deptree_to_simple_rst_tree() + # returns a SimpleRSTTree, vs a list of SimpleRSTTrees + # if allow_forest is True :-/ + bin_srtrees = [bin_srtrees] + # end of FIXME inconsistent API bin_rtrees = [SimpleRSTTree.to_binary_rst_tree(bin_srtree) for bin_srtree in bin_srtrees] except RstDtException as rst_e: print(rst_e) - if False: - print('\n'.join('{}: {}'.format(edu.text_span(), edu) - for edu in educe_edus[doc_name])) raise ctrees = bin_rtrees return ctrees -def get_spans(ctree): - """Get the spans of a constituency tree, except for the root node. - - This corresponds to the spans used in the PARSEVAL metric modified - for discourse, as described in (Marcu 2000) and implemented in - Joty's evaluation scripts. - - Each span is descried by a triplet (edu_span, nuclearity, relation). - """ - tnodes = [subtree.label() # was: educe.internalutil.treenode(subtree) - for root_child in ctree if isinstance(root_child, RSTTree) - for subtree in root_child.subtrees()] - spans = [(tn.edu_span, tn.nuclearity, tn.rel) - for tn in tnodes] - return spans - - def oracle_ctree_spans(dep_edges, att_edus): """Get the spans of the oracle ctree for a given dtree. @@ -180,5 +198,5 @@ def oracle_ctree_spans(dep_edges, att_edus): # to a forest of constituency trees oracle_ctrees = get_oracle_ctrees(dep_edges, att_edus) oracle_spans = list(itertools.chain.from_iterable( - [get_spans(oracle_ctree) for oracle_ctree in oracle_ctrees])) + [oracle_ctree.get_spans() for oracle_ctree in oracle_ctrees])) return oracle_spans diff --git a/attelo/parser/attach.py b/attelo/parser/attach.py index 0f2faab..4cd5bf1 100644 --- a/attelo/parser/attach.py +++ b/attelo/parser/attach.py @@ -4,7 +4,10 @@ You could also combine this with the label parser """ +from __future__ import print_function + from os import path as fp +import sys import joblib @@ -58,19 +61,34 @@ def fit(self, dpacks, targets, nonfixed_pairs=None, cache=None): return self dpacks, targets = self.dzip(for_attachment, dpacks, targets) - self._learner_attach.fit(dpacks, targets, - nonfixed_pairs=nonfixed_pairs) + + # WIP select only the nonfixed pairs + if nonfixed_pairs is not None: + dpacks = [dpack.selected(nf_pairs) + for dpack, nf_pairs in zip(dpacks, nonfixed_pairs)] + targets = [target[nf_pairs] + for target, nf_pairs in zip(targets, nonfixed_pairs)] + + self._learner_attach.fit(dpacks, targets) # save classifier, if necessary if cache_file is not None: - # print('\tsave {}'.format(cache_file)) joblib.dump(self._learner_attach, cache_file) return self def transform(self, dpack, nonfixed_pairs=None): attach_pack, _ = for_attachment(dpack, dpack.target) - weights_a = self._learner_attach.predict_score( - attach_pack, nonfixed_pairs=nonfixed_pairs) - dpack = self.multiply(dpack, attach=weights_a) + # WIP pass only nonfixed pairs to the classifier + if nonfixed_pairs is not None: + attach_pack = attach_pack.selected(nonfixed_pairs) + # end nonfixed_pairs + weights_a = self._learner_attach.predict_score(attach_pack) + # WIP overwrite only the attachment scores of non-fixed pairs + if nonfixed_pairs is not None: + scores = np.copy(dpack.graph.attach) + scores[nonfixed_pairs] = weights_a + else: + scores = weights_a + dpack = self.multiply(dpack, attach=scores) return dpack diff --git a/attelo/parser/intra.py b/attelo/parser/intra.py index fdc310d..738772c 100644 --- a/attelo/parser/intra.py +++ b/attelo/parser/intra.py @@ -11,7 +11,8 @@ from six import with_metaclass import numpy as np -from attelo.edu import (FAKE_ROOT_ID) +from attelo.cdu import CDU +from attelo.edu import (FAKE_ROOT_ID, edu_id2num) from attelo.table import (DataPack, Graph, UNRELATED, @@ -45,6 +46,59 @@ def fmap(self, fun): return IntraInterPair(intra=fun(self.intra), inter=fun(self.inter)) + +def _for_intra_cdu(dpack, target, grp, unrelated, intra_tgts): + """Helper to for_intra to contain CDU-specific code. + + Parameters + ---------- + dpack : DataPack + DataPack + target : TODO + TODO + grp : :obj:`dict` of (str, str) + Map from EDU identifier to subgroup identifier. + unrelated : int + Number of the "unrelated" label for this datapack. + intra_tgts : :obj:`dict` of (str, set(str)) + Map each subgroup (identifier) to the set of its EDUs + (identifiers) that have incoming edges whose source is in the + same subgroup. + """ + if not dpack.cdu_pairings: + # no CDU pairings: fail early + all_heads_cdu = [] + inter_links_cdu = [] + new_cdu_target = None + return all_heads_cdu, inter_links_cdu, new_cdu_target + + all_heads_cdu = [] + for i, (du1, du2) in enumerate(dpack.cdu_pairings): + src = (du1.members[0] if isinstance(du1, CDU) else du1.id) + tgt = (du2.members[0] if isinstance(du2, CDU) else du2.id) + if (src == FAKE_ROOT_ID + and tgt not in intra_tgts[grp[tgt]]): + # leftmost member of du2 is an intra root => + # keep (ROOT, leftmost member of du2) + all_heads_cdu.append(i) + inter_links_cdu = [] + for i, (du1, du2) in enumerate(dpack.cdu_pairings): + src = (du1.members[0] if isinstance(du1, CDU) else du1.id) + tgt = (du2.members[0] if isinstance(du2, CDU) else du2.id) + if (src != FAKE_ROOT_ID + and grp[src] != grp[tgt] + and dpack.cdu_target[i] != unrelated): + # inter link should be removed + inter_links_cdu.append(i) + if dpack.cdu_target is not None: + new_cdu_target = np.copy(dpack.cdu_target) + new_cdu_target[all_heads_cdu] = dpack.label_number('ROOT') + new_cdu_target[inter_links_cdu] = unrelated + else: + new_cdu_target = None + return all_heads_cdu, inter_links_cdu, new_cdu_target + + def for_intra(dpack, target): """Adapt a datapack to intrasentential decoding. @@ -58,9 +112,9 @@ def for_intra(dpack, target): Returns ------- dpack : DataPack - + DataPack. target : array(int) - + TODO. """ # map EDUs to subgroup ids ; intra = pairs of EDUs with same subgroup id grp = {e.id: e.subgrouping for e in dpack.edus} @@ -68,10 +122,12 @@ def for_intra(dpack, target): unrelated = dpack.label_number(UNRELATED) intra_tgts = defaultdict(set) for i, (edu1, edu2) in enumerate(dpack.pairings): - if (grp[edu1.id] == grp[edu2.id] + if (edu1.id != FAKE_ROOT_ID + and grp[edu1.id] == grp[edu2.id] and target[i] != unrelated): # edu2 has an incoming relation => not an (intra) root intra_tgts[grp[edu2.id]].add(edu2.id) + # pick out the (fakeroot, edu) pairs where edu does not have # incoming intra edges all_heads = [i for i, (edu1, edu2) in enumerate(dpack.pairings) @@ -82,6 +138,10 @@ def for_intra(dpack, target): if (edu1.id != FAKE_ROOT_ID and grp[edu1.id] != grp[edu2.id] and target[i] != unrelated)] + # 2016-07-29 CDUs + all_heads_cdu, inter_links_cdu, new_cdu_target = _for_intra_cdu( + dpack, target, grp, unrelated, intra_tgts) + # end CDUs # update datapack and target accordingly new_target = np.copy(dpack.target) @@ -98,6 +158,12 @@ def for_intra(dpack, target): data=dpack.data, target=new_target, ctarget=new_ctarget, + # 2016-07-28 WIP CDUs + cdus=dpack.cdus, + cdu_pairings=dpack.cdu_pairings, + cdu_data=dpack.cdu_data, + cdu_target=new_cdu_target, + # end WIP CDUs labels=dpack.labels, vocab=dpack.vocab, graph=dpack.graph) @@ -332,7 +398,7 @@ def _for_inter_fit(self, dpack, target): def fit(self, dpacks, targets, cache=None): caches = self._split_cache(cache) - # print('intra.fit') + # print('intra.fit') # DEBUG if dpacks: dpacks_intra, targets_intra = self.dzip(self._for_intra_fit, dpacks, targets) @@ -354,7 +420,7 @@ def fit(self, dpacks, targets, cache=None): targets_spacks.extend(target_spacks) self._parsers.intra.fit(dpacks_spacks, targets_spacks, cache=caches.intra) - # print('inter.fit') + # print('inter.fit') # DEBUG if dpacks: dpacks_inter, targets_inter = self.dzip(self._for_inter_fit, dpacks, targets) @@ -670,15 +736,6 @@ def merged_lbl(i): return dpack -# small helper for the FrontierToHeadParser -def edu_id2num(edu_id): - """Get the number of an EDU""" - edu_num = (int(edu_id.rsplit('_', 1)[1]) - if edu_id != FAKE_ROOT_ID - else 0) - return edu_num - - class FrontierToHeadParser(IntraInterParser): """Intra/inter parser in which sentence recombination consists of parsing with edges from the frontier of sentential subtree to sentence diff --git a/attelo/parser/label.py b/attelo/parser/label.py index 3e2a89b..1361778 100644 --- a/attelo/parser/label.py +++ b/attelo/parser/label.py @@ -2,7 +2,10 @@ Labelling """ +from __future__ import print_function + from os import path as fp +import sys import joblib import numpy as np @@ -68,18 +71,35 @@ def fit(self, dpacks, targets, nonfixed_pairs=None, cache=None): dpacks, targets = self.dzip(attached_only, dpacks, targets) dpacks, targets = self.dzip(for_labelling, dpacks, targets) - self._learner.fit(dpacks, targets, nonfixed_pairs=nonfixed_pairs) + # WIP select only the nonfixed pairs + if nonfixed_pairs is not None: + dpacks = [dpack.selected(nf_pairs) + for dpack, nf_pairs in zip(dpacks, nonfixed_pairs)] + targets = [target[nf_pairs] + for target, nf_pairs in zip(targets, nonfixed_pairs)] + + self._learner.fit(dpacks, targets) # save classifier, if necessary if cache_file is not None: - # print('\tsave {}'.format(cache_file)) joblib.dump(self._learner, cache_file) + return self def transform(self, dpack, nonfixed_pairs=None): - dpack, _ = for_labelling(dpack, dpack.target) - weights_l = self._learner.predict_score( - dpack, nonfixed_pairs=nonfixed_pairs) - dpack = self.multiply(dpack, label=weights_l) + label_pack, _ = for_labelling(dpack, dpack.target) + # WIP don't pass the fixed pairs to the classifier + if nonfixed_pairs is not None: + label_pack = label_pack.selected(nonfixed_pairs) + + weights_l = self._learner.predict_score(label_pack) + # WIP overwrite only the labelling scores of non-fixed pairs + if nonfixed_pairs is not None: + lbl_scores = np.copy(dpack.graph.label) + lbl_scores[nonfixed_pairs] = weights_l + else: + lbl_scores = weights_l + + dpack = self.multiply(dpack, label=lbl_scores) return dpack diff --git a/attelo/parser/pipeline.py b/attelo/parser/pipeline.py index 2262650..3ca5c58 100644 --- a/attelo/parser/pipeline.py +++ b/attelo/parser/pipeline.py @@ -7,6 +7,10 @@ from __future__ import absolute_import, print_function +# FIXME: look into using sklearn.pipeline.Pipeline +# I wasn't too successful last time + +from attelo.io import Torpor from .interface import Parser @@ -17,7 +21,7 @@ class Pipeline(Parser): fitted independently of each other. Steps should be a tuple of names and parsers, just like - in sklearn. + in sklearn.pipeline.Pipeline. Parameters ---------- diff --git a/attelo/parser/same_unit.py b/attelo/parser/same_unit.py new file mode 100644 index 0000000..92b9dd4 --- /dev/null +++ b/attelo/parser/same_unit.py @@ -0,0 +1,396 @@ +"""Preprocessor to detect and link fragments of EDus ("same-unit"). +""" + +from __future__ import print_function + +import csv +from os import path as fp +import os + +import joblib +import numpy as np + +from attelo.edu import edu_id2num, FAKE_ROOT_ID +from attelo.table import UNKNOWN, DataPack, Graph +from attelo.learning.interface import AttachClassifier +from attelo.learning.local import SklearnClassifier +from attelo.learning.oracle import AttachOracle +from attelo.parser.attach import AttachClassifierWrapper +from attelo.parser.full import AttachTimesBestLabel +from attelo.parser.label import LabelClassifierWrapper +from attelo.parser.interface import Parser +from attelo.parser.pipeline import Pipeline + + +SAME_UNIT = "same-unit" + + +def for_attachment_same_unit(dpack, target): + """Adapt a datapack to the task of linking fragments of EDUs. + + This is modelled as an attachment task restricted to the "same-unit" + label. + + This could involve: + * selecting some of the features (all for now, but may change in the + future) + * modifying the features/labels in some way: we currently binarise + labels to {-1 ; 1} for UNRELATED and not-UNRELATED respectively. + + Parameters + ---------- + dpack: DataPack + Original datapack + target: array(int) + Original targets + + Returns + ------- + dpack: DataPack + Transformed datapack, with binary labels + + target: array(int) + Transformed targets, with binary labels + """ + su_idx = dpack.label_number(SAME_UNIT) + dpack = DataPack(edus=dpack.edus, + pairings=dpack.pairings, + data=dpack.data, + target=np.where(dpack.target == su_idx, 1, -1), + ctarget=dpack.ctarget, # ctree target ; old WIP + # 2016-07-28 WIP CDUs + cdus=dpack.cdus, + cdu_pairings=dpack.cdu_pairings, + cdu_data=dpack.cdu_data, + cdu_target=dpack.cdu_target, + # end WIP CDUs + labels=[UNKNOWN, SAME_UNIT], + vocab=dpack.vocab, + graph=dpack.graph) + target = np.where(target == su_idx, 1, -1) + return dpack, target + + +def right_intra_idc(dpack): + """Get the indices of right-attachment, intra-sentential pairings. + + Parameters + ---------- + dpack: DataPack + Datapack + + Returns + ------- + res: array of integers + Indices of right, intra-sentential candidates in dpack.pairings. + """ + edu_id2sent = {e.id: e.subgrouping for e in dpack.edus} + res = [i for i, (edu1, edu2) in enumerate(dpack.pairings) + if (edu1.id != FAKE_ROOT_ID and + edu_id2sent[edu1.id] == edu_id2sent[edu2.id] and + edu_id2num(edu1.id) < edu_id2num(edu2.id))] + return res + + +class SameUnitClassifierWrapper(Parser): + """ + Parser that extracts attachments weights from a "same-unit" + classifier. + + This parser is really meant to be used in conjunction with + other parsers downstream that make use of these weights. + + If you use it in standalone mode, it will just provide the + standard unknown prediction everywhere + + Notes + ----- + *Cache keys* + + * attach: attachment model path + """ + def __init__(self, learner_su): + """ + Parameters + ---------- + learner_su : AttachClassifier + Learner to use for prediction of "Same-Unit". + """ + self._learner_su = learner_su + + def fit(self, dpacks, targets, nonfixed_pairs=None, cache=None): + """ + Extract whatever models or other information from the multipack + that is necessary to make the parser operational + + Parameters + ---------- + dpacks: list of DataPack + List of datapacks, one per "stuctured instance" (ex: + document or sentence for RST, document, dialogue or + turn for STAC). + + targets: list of array of int + List of arrays of gold labels, one array per structured + instance, then one integer (label) per candidate edge. + + nonfixed_pairs: list of array of int + List of arrays of indexes, corresponding to the non-fixed + candidate edges in each instance. + + cache: TODO + TODO + + Returns + ------- + self: SameUnitClassifierWrapper + Fitted self. + """ + cache_file = (cache.get('su') if cache is not None + else None) + # load cached classifier, if it exists + if cache_file is not None and fp.exists(cache_file): + # print('\tload {}'.format(cache_file)) + self._learner_su = joblib.load(cache_file) + return self + + dpacks, targets = self.dzip(for_attachment_same_unit, + dpacks, targets) + + # WIP filter: pass only nonfixed, right-attachment, intra-sentential + # pairs to the classifier + ri_pairs = [right_intra_idc(x) for x in dpacks] + if nonfixed_pairs is not None: + nf_pairs = [list(np.intersect1d(rip, nfp)) for rip, nfp + in zip(ri_pairs, nonfixed_pairs)] + else: + nf_pairs = ri_pairs + + dpacks = [dpack.selected(nfp) for dpack, nfp + in zip(dpacks, nf_pairs)] + targets = [target[nfp] for target, nfp + in zip(targets, nf_pairs)] + # end filter + + self._learner_su.fit(dpacks, targets) + # save classifier, if necessary + if cache_file is not None: + # print('\tsave {}'.format(cache_file)) + joblib.dump(self._learner_su, cache_file) + return self + + def _dump_frag_edus(self, dpack, scores_pred, positive_mask, su_pred, + verbose=0): + """Helper to dump fragmented EDUs predicted by this classifier. + + Called by `transform()`. + """ + # edus[1:] to skip the fake root, as its grouping is None + doc_names = set(x.grouping for x in dpack.edus[1:]) + assert len(doc_names) == 1 + doc_name = list(doc_names)[0] + # verbose + if verbose: + print('Predicted same-unit in', doc_name) + for i, (su_score_pred, pair) in enumerate(zip( + scores_pred[positive_mask], + [dpack.pairings[i] for i in su_pred]), start=1): + print('{:.2f}'.format(su_score_pred), pair[0]) + print(' ', pair[1]) + + # dump + out_dir = 'TMP_same_unit' # FIXME + if not os.path.exists(out_dir): + os.makedirs(out_dir) + # avoid confusing true and predicted Same-Unit + if isinstance(self._learner_su, AttachOracle): + fn_ext = 'deps_true' + else: + fn_ext = 'deps_pred' + + # FIXME this assumes the first pass is on the whole doc, not + # a subset like sentence dpack as in (intra/inter) ; hence + # this works as expected if and only if the first parser run + # in the harness is a "global" one + fpath_su = os.path.join( + out_dir, + '{}.relations.same-unit.{}'.format(doc_name, fn_ext)) + if not os.path.exists(fpath_su): + frag_edu_pairs = [dpack.pairings[i] for i in su_pred] + frag_edu_pairs = [(src.id, tgt.id) for src, tgt + in frag_edu_pairs] + with open(fpath_su, 'wb') as f_out: + su_writer = csv.writer(f_out, dialect=csv.excel_tab) + for i, frag_edu_members in enumerate( + frag_edu_pairs, start=1): + frag_edu_id = doc_name + '_frag' + str(i) + su_writer.writerow( + [frag_edu_id] + list(frag_edu_members)) + + def transform(self, dpack, nonfixed_pairs=None, verbose=0): + """ + Its main effect is to update the arrays of + scores for attachment and labelling in `dpack`, for all + candidate edges where a "same-unit" has been predicted. + This is a sort of side-effect, so one should be careful + about it when using this function. + """ + + if dpack.graph is None: + # SklearnSameUnitClassifier.predict_score requires a + # weighted datapack + dpack = self.multiply(dpack) + + # we'll update these copies + scores_att = np.copy(dpack.graph.attach) + scores_lbl = np.copy(dpack.graph.label) + + # su_pack, _ = for_attachment_same_unit(dpack, dpack.target) + su_pack = dpack + # WIP filter: pass only nonfixed, right-attachment, intra-sentential + # pairs to the classifier + ri_pairs = right_intra_idc(su_pack) + if nonfixed_pairs is not None: + nf_pairs = list(np.intersect1d(ri_pairs, nonfixed_pairs)) + else: + nf_pairs = ri_pairs + + if not nf_pairs: + # no prediction to make (ex: doc with 2 EDUs, 1 sentence each): + # return (copies of) the original scores + # DIRTY this assumes that su_pack.edus[0] is the first real + # EDU, not the fake root + if verbose: + print('no same-unit prediction where ', su_pack.edus[0].id) + return dpack + + su_pack = su_pack.selected(nf_pairs) + # end filter + + scores_pred = self._learner_su.predict_score(su_pack) + + # positive_mask is an array of booleans: True if the + # corresponding pair has been predicted as "same-unit", + # False otherwise + positive_mask = (scores_pred > 0.5 + if self._learner_su.can_predict_proba + else scores_pred > 0) + # get the absolute indices of pairs for which same-unit has been + # predicted + su_pred = np.array(nf_pairs)[positive_mask] + if verbose: + # WIP 2016-08-25 dump predicted frag EDUs + self._dump_frag_edus(dpack, scores_pred, positive_mask, su_pred) + + # update the lines of predicted "same-unit" in the matrices of + # scores for attachment and labels: + # * attachment: set the score to the predicted score for + # "same-unit", + # * label: set the score for "same-unit" to the predicted score, + # set the scores for other labels to 0. + scores_att[su_pred] = scores_pred[positive_mask] + # + update_lbl = np.zeros(scores_lbl[su_pred].shape, dtype=float) + su_idx = dpack.label_number(SAME_UNIT) + update_lbl[:, su_idx] = scores_pred[positive_mask] + scores_lbl[su_pred] = update_lbl + + # update dpack graph + graph = dpack.graph.tweak(attach=scores_att, + label=scores_lbl) + dpack = dpack.set_graph(graph) + return dpack + + +class SameUnitJointPipeline(Pipeline): + """Same-unit preprocessor then JointPipeline. + + Predicted "same-unit" are used to generate new instances. + The prediction score from the same-unit preprocessor are used to + overwrite the corresponding attachment and labelling scores in the + arrays of attachment and labelling scores, before the product of + scores is computed. + + Parameters + ---------- + learner_su: SameUnitClassifier + + learner_attach: AttachClassifier + + learner_label: LabelClassifier + + decoder: Decoder + + Notes + ----- + *Cache keys* + + * attach: attach model path + * label: label model path + * su: same-unit model path + """ + def __init__(self, learner_su, learner_attach, learner_label, decoder): + if not learner_attach.can_predict_proba: + raise ValueError('Attachment model does not know how to predict ' + 'probabilities.') + if not learner_label.can_predict_proba: + raise ValueError('Relation labelling model does not ' + 'know how to predict probabilities') + if not learner_su.can_predict_proba: + raise ValueError('Same-Unit model does not ' + 'know how to predict probabilities') + + steps = [ + ('same-unit weights', SameUnitClassifierWrapper(learner_su)), + # + ('attach weights', AttachClassifierWrapper(learner_attach)), + ('label weights', LabelClassifierWrapper(learner_label)), + ('attach x best label', AttachTimesBestLabel()), + ('decoder', decoder) + ] + super(SameUnitJointPipeline, self).__init__(steps=steps) + + +class JointSameUnitPipeline(Pipeline): + """JointPipeline with an extra step to predict "same-unit". + + The scores of predicted "same-unit" are used to overwrite the + corresponding attachment and labelling scores in the arrays of + attachment and labelling scores, before the product of scores is + computed. + + Parameters + ---------- + learner_attach: AttachClassifier + + learner_label: LabelClassifier + + learner_su: SameUnitClassifier + + decoder: Decoder + + Notes + ----- + *Cache keys* + + * attach: attach model path + * label: label model path + * su: same-unit model path + """ + def __init__(self, learner_attach, learner_label, learner_su, decoder): + if not learner_attach.can_predict_proba: + raise ValueError('Attachment model does not know how to predict ' + 'probabilities.') + if not learner_label.can_predict_proba: + raise ValueError('Relation labelling model does not ' + 'know how to predict probabilities') + if not learner_su.can_predict_proba: + raise ValueError('Same-Unit model does not ' + 'know how to predict probabilities') + + steps = [('attach weights', AttachClassifierWrapper(learner_attach)), + ('label weights', LabelClassifierWrapper(learner_label)), + ('same-unit weights', SameUnitClassifierWrapper(learner_su)), + ('attach x best label', AttachTimesBestLabel()), + ('decoder', decoder)] + super(JointSameUnitPipeline, self).__init__(steps=steps) diff --git a/attelo/score.py b/attelo/score.py index 1d05255..3744872 100644 --- a/attelo/score.py +++ b/attelo/score.py @@ -2,6 +2,8 @@ Scoring decoding results ''' +from __future__ import print_function + from collections import (defaultdict, namedtuple) import itertools @@ -11,14 +13,11 @@ # WIP from educe.rst_dt.annotation import _binarize from educe.rst_dt.corpus import RstRelationConverter, RELMAP_112_18_FILE +from educe.rst_dt.metrics.rst_parseval import LBL_FNS, rst_parseval_report # end WIP -from .table import (UNRELATED, - attached_only, - get_label_string) -from .metrics.classification_structured import precision_recall_fscore_support -from .metrics.constituency import LBL_FNS -from .metrics.util import get_oracle_ctrees, get_spans, oracle_ctree_spans +from .table import UNRELATED, attached_only, get_label_string +from .metrics.util import get_oracle_ctrees, oracle_ctree_spans # pylint: disable=too-few-public-methods @@ -225,7 +224,7 @@ def score_cspans(dpacks, dpredictions, coarse_rels=True, binary_trees=True, # end WIP # spans of the gold constituency trees ctree_spans_golds = [list(itertools.chain.from_iterable( - get_spans(ctg) for ctg in ctree_gold)) + ctg.get_spans() for ctg in ctree_gold)) for ctree_gold in ctree_golds] # spans of the predicted oracle constituency trees edges_preds = [[(edu1, edu2, rel) @@ -236,6 +235,27 @@ def score_cspans(dpacks, dpredictions, coarse_rels=True, binary_trees=True, for edges_pred, att_pack in zip(edges_preds, att_packs)] + ctree_true = ctree_golds # yerk + ctree_pred = [get_oracle_ctrees(edges_pred, att_pack.edus, + allow_forest=False) + for edges_pred, att_pack + in zip(edges_preds, att_packs)] + # 2016-10-02 force one ctree per doc ; we need to reconsider when we + # do doc-level eval + for ct_true in ctree_true: + if len(ct_true) > 1: + raise NotImplementedError( + "Currently unable to handle multiple ctrees per doc") + ctree_true = [ct_true[0] for ct_true in ctree_true] + for ct_pred in ctree_pred: + if len(ct_pred) > 1: + raise NotImplementedError( + "Currently unable to handle multiple ctrees per doc") + ctree_pred = [ct_pred[0] for ct_pred in ctree_pred] + # end force one ctree per doc + + print(rst_parseval_report(ctree_true, ctree_pred)) + # FIXME replace loop with attelo.metrics.constituency.XXX cnts = [] for metric_type, lbl_fn in LBL_FNS: diff --git a/attelo/table.py b/attelo/table.py index a025a6b..da206aa 100644 --- a/attelo/table.py +++ b/attelo/table.py @@ -9,6 +9,7 @@ import numpy as np import scipy.sparse +from .cdu import CDU from .edu import FAKE_ROOT_ID from .util import concat_l @@ -125,6 +126,12 @@ class DataPack(namedtuple('DataPack', 'data', 'target', 'ctarget', + # 2016-07-28 WIP CDUs for frag EDUs + 'cdus', + 'cdu_pairings', + 'cdu_data', + 'cdu_target', + # end WIP 'labels', 'vocab', 'graph'])): @@ -180,7 +187,9 @@ def __len__(self): # pylint: disable=too-many-arguments @classmethod - def load(cls, edus, pairings, data, target, ctarget, labels, vocab): + def load(cls, edus, pairings, data, target, ctarget, + cdus, cdu_pairings, cdu_data, cdu_target, + labels, vocab): ''' Build a data pack and run some sanity checks (see :py:method:sanity_check') @@ -188,13 +197,14 @@ def load(cls, edus, pairings, data, target, ctarget, labels, vocab): :rtype: :py:class:`DataPack` ''' - pack = cls(edus=edus, - pairings=pairings, - data=data, - target=target, + pack = cls(edus=edus, pairings=pairings, + data=data, target=target, ctarget=ctarget, - labels=labels, - vocab=vocab, + # WIP CDUs + cdus=cdus, cdu_pairings=cdu_pairings, + cdu_data=cdu_data, cdu_target=cdu_target, + # end CDUs + labels=labels, vocab=vocab, graph=None) pack.sanity_check() return pack @@ -212,15 +222,40 @@ def vstack(cls, dpacks): if not dpacks: raise ValueError('need non-empty list of datapacks') dzero = dpacks[0] + + # merge ctargets + new_ctarget = defaultdict(list) + for d in dpacks: + for grp_name, ctgt in d.ctarget.items(): + new_ctarget[grp_name].append(ctgt) + # end merge ctargets + + # CDUs + if any(d.cdus for d in dpacks): + cdus = concat_l(d.cdus for d in dpacks) + cdu_pairings = concat_l(d.cdu_pairings for d in dpacks) + cdu_data = scipy.sparse.vstack(d.cdu_data for d in dpacks) + cdu_target = (np.concatenate([d.cdu_target for d in dpacks + if d.cdu_target is not None]) + if any(d.cdu_target is not None for d in dpacks) + else None) + else: + cdus = None + cdu_pairings = None + cdu_data = None + cdu_target = None + # end CDUs return DataPack(edus=concat_l(d.edus for d in dpacks), pairings=concat_l(d.pairings for d in dpacks), data=scipy.sparse.vstack(d.data for d in dpacks), target=np.concatenate([d.target for d in dpacks]), - ctarget={grp_name: list(itertools.chain.from_iterable( - d.ctarget.get(grp_name, []) for d in dpacks)) - for grp_name in - set(itertools.chain.from_iterable( - d.ctarget.keys() for d in dpacks))}, + ctarget=new_ctarget, + # CDUs + cdus=cdus, + cdu_pairings=cdu_pairings, + cdu_data=cdu_data, + cdu_target=cdu_target, + # end CDUs labels=dzero.labels, vocab=dzero.vocab, graph=Graph.vstack(d.graph for d in dpacks)) @@ -282,6 +317,36 @@ def sanity_check(self): self._check_target() self._check_table_shape() + def _selected_cdu(self, sel_pairings): + """Helper to selected() for CDUs""" + # 2016-07-28 restrict pairings from/to CDUs: keep only those + # that correspond to selected pairings from/to the first member + # of a CDU + sel_pairings_id_set = set((src.id, tgt.id) + for src, tgt in sel_pairings) + # get the EDU pairings that correspond to each CDU pairing + edu_pairings_cdu = [ + (src.members[0] if isinstance(src, CDU) else src.id, + tgt.members[0] if isinstance(tgt, CDU) else tgt.id) + for src, tgt in self.cdu_pairings + ] + # filter against the set of selected EDU pairings + sel_cdu_indices = [i for i, x in enumerate(edu_pairings_cdu) + if x in sel_pairings_id_set] + sel_cdu_pairings = [self.cdu_pairings[x] for x in sel_cdu_indices] + sel_cdus_ = set() + for du1, du2 in sel_cdu_pairings: + if isinstance(du1, CDU): + sel_cdus_.add(du1) + if isinstance(du2, CDU): + sel_cdus_.add(du2) + sel_cdus = [x for x in self.cdus if x in sel_cdus_] + sel_cdu_data = (self.cdu_data[sel_cdu_indices] + if self.cdu_data is not None and sel_cdu_indices + else None) + sel_cdu_targets = np.take(self.cdu_target, sel_cdu_indices) + return sel_cdus, sel_cdu_pairings, sel_cdu_data, sel_cdu_targets + def selected(self, indices): ''' Return only the items in the specified rows @@ -311,11 +376,28 @@ def selected(self, indices): graph = None else: graph = self.graph.selected(indices) + + # CDUs: restrict pairings from/to CDUs to selected pairings + # from/to the first member of each CDU + if self.cdus: + sel_cdus, sel_cdu_pairings, sel_cdu_data, sel_cdu_targets = self._selected_cdu(sel_pairings) + else: + sel_cdus = self.cdus + sel_cdu_pairings = self.cdu_pairings + sel_cdu_data = self.cdu_data + sel_cdu_targets = self.cdu_target + # end WIP CDU return DataPack(edus=sel_edus, pairings=sel_pairings, data=sel_data, target=sel_targets, ctarget=sel_ctargets, # WIP + # 2016-07-28 WIP on CDUs + cdus=sel_cdus, + cdu_pairings=sel_cdu_pairings, + cdu_data=sel_cdu_data, + cdu_target=sel_cdu_targets, + # end WIP labels=sel_labels, vocab=self.vocab, graph=graph) @@ -351,6 +433,12 @@ def set_graph(self, graph): data=self.data, target=self.target, ctarget=self.ctarget, + # 2016-07-29 WIP CDUs + cdus=self.cdus, + cdu_pairings=self.cdu_pairings, + cdu_data=self.cdu_data, + cdu_target=self.cdu_target, + # end CDUs labels=self.labels, vocab=self.vocab, graph=graph) @@ -504,6 +592,12 @@ def for_attachment(dpack, target): data=dpack.data, target=np.where(dpack.target == unrelated, -1, 1), ctarget=dpack.ctarget, # WIP + # 2016-07-28 WIP CDUs + cdus=dpack.cdus, + cdu_pairings=dpack.cdu_pairings, + cdu_data=dpack.cdu_data, + cdu_target=dpack.cdu_target, + # end WIP labels=[UNKNOWN, UNRELATED], vocab=dpack.vocab, graph=dpack.graph) diff --git a/setup.py b/setup.py index e671e20..174cd4c 100644 --- a/setup.py +++ b/setup.py @@ -14,12 +14,12 @@ scripts=["scripts/attelo"], install_requires=['depparse', 'enum34', - 'joblib', + 'joblib >= 0.9.4', 'mock', 'nltk', 'numpy', 'pydot', - 'scikit-learn >= 0.17', + 'scikit-learn >= 0.17.1', 'six', 'scipy >= 0.14.0', 'tabulate'])