MAINT catchup with changes in API of educe and attelo

intake/create-glozz-aam.py

@@ -20,7 +20,7 @@
 import sys
 import xml.etree.ElementTree as ET
 
-import educe.stac.util.csv as stac_csv
+import educe.stac.util.stac_csv_format as stac_csv
 
 # ---------------------------------------------------------------------
 # template

intake/csvtoglozz.py

@@ -54,7 +54,7 @@
 import time
 
 from educe.stac.util.prettifyxml import prettify
-from educe.stac.util.csv import Turn
+from educe.stac.util.stac_csv_format import Turn
 
 
 class Span(namedtuple('Span', 'left right')):

intake/soclogtocsv.py

@@ -38,7 +38,7 @@
 import string
 import sys
 
-from educe.stac.util import csv as stac_csv
+from educe.stac.util import stac_csv_format as stac_csv
 
 # TODO write tests for these
 TEST1 = ("2011:10:10:17:46:57:481:+0100:GAME-TEXT-MESSAGE:"

requirements.txt

@@ -1,3 +1,5 @@
--e git+https://github.com/irit-melodi/educe.git#egg=educe
--e git+https://github.com/irit-melodi/attelo.git#egg=attelo
+# -e git+https://github.com/irit-melodi/educe.git#egg=educe
+-e /home/mmorey/melodi/educe
+# -e git+https://github.com/irit-melodi/attelo.git#egg=attelo
+-e /home/mmorey/melodi/attelo
 -e .

segmentation/normalise-csv

@@ -17,7 +17,7 @@ import os
 import os.path
 import sys
 
-import educe.stac.util.csv
+import educe.stac.util.stac_csv_format as stac_csv
 
 if len(sys.argv) == 3:
     filename_in  = sys.argv[1]
@@ -30,7 +30,7 @@ def normalise_file(filename_in, filename_out):
     with open(filename_in, 'rb') as infile:
         with open(filename_out, 'wb') as outfile:
             reader = csv.reader(infile,  delimiter='\t')
-            writer = educe.stac.util.csv.mk_plain_writer(outfile)
+            writer = stac_csv.mk_plain_writer(outfile)
             for row in reader:
                 writer.writerow(row)
 

segmentation/simple-segments

@@ -21,7 +21,7 @@ import re
 import sys
 
 import segmentation
-import educe.stac.util.csv
+import educe.stac.util.stac_csv_format as stac_csv
 
 def segment_row(t):
     segments = [segmentation.span_text(t,sp) for sp in segmentation.segment(t)]
@@ -64,11 +64,11 @@ else:
     job=lambda t:t
 
 with open(filename_in, 'rb') as infile:
-    reader = educe.stac.util.csv.mk_csv_reader(infile)
+    reader = stac_csv.mk_csv_reader(infile)
     if args.csv:
         # csv library has built-in utf-8 encoding
         with open(args.output_file, 'wb') as outfile:
-            writer = educe.stac.util.csv.mk_csv_writer(outfile)
+            writer = stac_csv.mk_csv_writer(outfile)
             writer.writeheader()
             for row in reader:
                 writer.writerow(replace_text(job,row))

stac/attelo_out.py

@@ -66,11 +66,11 @@ def split_id(anno_id):
     local suffix
     """
     # WIP match a widespread format: doc_timestamp_anno
-    m = DOC_TIMESTAMP.match(anno_id)
-    if m is not None:
-        doc = m.group('doc_id')
-        subdoc = m.group('subdoc_id')
-        suffix = m.group('loc_id')
+    ts_match = DOC_TIMESTAMP.match(anno_id)
+    if ts_match is not None:
+        doc = ts_match.group('doc_id')
+        subdoc = ts_match.group('subdoc_id')
+        suffix = ts_match.group('loc_id')
     else:
         doc, subdoc, suffix = anno_id.split('_', 2)
     return ((doc, subdoc), suffix)

stac/harness/cmd/gather.py

@@ -31,14 +31,14 @@ def config_argparser(psr):
                      default=False, action="store_true",
                      help="only gather test data")
     psr.add_argument('--strip-mode',
-                    choices=['head', 'broadcast', 'custom'],
-                    default='head',
-                    help='CDUs stripping method')
+                     choices=['head', 'broadcast', 'custom'],
+                     default='head',
+                     help='CDUs stripping method')
     psr.set_defaults(func=main)
 
 
 def extract_features(corpus, output_dir,
-                     vocab_path=None, strip_mode=None):
+                     vocab_path=None, label_path=None, strip_mode=None):
     """Extract features for a corpus, dump the instances.
 
     Run feature extraction for a particular corpus; and store the
@@ -50,15 +50,21 @@ def extract_features(corpus, output_dir,
 
     Parameters
     ----------
-    corpus: filepath
+    corpus : filepath
         Selected corpus
-    output_dir: filepath
+
+    output_dir : filepath
         Folder where instances will be dumped
-    vocab_path: filepath
+
+    vocab_path : filepath
         Vocabulary to load for feature extraction (needed if extracting
         test data; must ensure we have the same vocab in test as we'd
         have in training)
-    strip_mode: one of {'head', 'broadcast', 'custom'}
+
+    label_path : filepath
+        Path to a file that describes the labelset.
+
+    strip_mode : one of {'head', 'broadcast', 'custom'}
         Method to strip CDUs
     """
     # TODO: perhaps we could just directly invoke the appropriate
@@ -70,6 +76,8 @@ def extract_features(corpus, output_dir,
            "--anno", ANNOTATORS]
     if vocab_path is not None:
         cmd.extend(['--vocabulary', vocab_path])
+    if label_path is not None:
+        cmd.extend(['--labels', label_path])
     if strip_mode is not None:
         cmd.extend(['--strip-mode', strip_mode])
     call(cmd)
@@ -87,17 +95,28 @@ def main(args):
         tdir = latest_tmp()
     else:
         tdir = current_tmp()
-        extract_features(TRAINING_CORPUS, tdir, strip_mode=args.strip_mode)
+
+    # 2016-09-01 put data files in {tdir}/data
+    tdir_data = os.path.join(tdir, 'data')
+    if not os.path.exists(tdir_data):
+        os.makedirs(tdir_data)
+
+    if not args.skip_training:
+        extract_features(TRAINING_CORPUS, tdir_data,
+                         strip_mode=args.strip_mode)
 
     if TEST_CORPUS is not None:
-        vocab_path = fp.join(tdir,
+        vocab_path = fp.join(tdir_data,
                              (fp.basename(TRAINING_CORPUS) +
                               '.relations.sparse.vocab'))
-        extract_features(TEST_CORPUS, tdir,
+        label_path = fp.join(tdir_data,
+                             (fp.basename(TRAINING_CORPUS) +
+                              '.relations.labels'))
+        extract_features(TEST_CORPUS, tdir_data,
                          vocab_path=vocab_path,
                          strip_mode=args.strip_mode)
 
-    with open(os.path.join(tdir, "versions-gather.txt"), "w") as stream:
+    with open(os.path.join(tdir_data, "versions-gather.txt"), "w") as stream:
         call(["pip", "freeze"], stdout=stream)
 
     if not args.skip_training:

stac/harness/cmd/model.py

@@ -62,10 +62,10 @@ def _do_corpus(hconf):
     paths = hconf.mpack_paths(test_data=False)
     if not fp.exists(paths[0]):
         exit_ungathered()
-    mpack = load_multipack(paths[0],
-                           paths[1],
-                           paths[2],
-                           paths[3],
+    mpack = load_multipack(paths['edu_input'], paths['pairings'],
+                           paths['features'],
+                           paths['vocab'], paths['labels'],
+                           file_split='corpus',  # WIP
                            verbose=True)
     dconf = DataConfig(pack=mpack,
                        folds=None)

stac/harness/cmd/parse.py

@@ -16,8 +16,7 @@
 
 from ..local import (CORENLP_SERVER_DIR, CORENLP_ADDRESS,
                      TAGGER_JAR, LEX_DIR,
-                     DIALOGUE_ACT_LEARNER,
-                     EVALUATIONS)
+                     DIALOGUE_ACT_LEARNER)
 from ..pipeline import\
     (StandaloneParser,
      Stage, run_pipeline,
@@ -53,14 +52,10 @@ def _soclog_to_csv(lconf, log):
     convert to the csv format more familiar to the
     rest of the scripts
     """
-    turns_path = lconf.tmp("turns")
-    with open(turns_path, "w") as turns_file:
-        lconf.pyt("txt2csv/extract_turns.py", lconf.soclog,
-                  stdout=turns_file)
-    lconf.pyt("txt2csv/extract_annot.py", turns_path,
-              stdout=log)
-    os.rename(turns_path + "csv", unseg_path(lconf))
-    os.unlink(turns_path)
+    lconf.pyt("intake/soclogtocsv.py",
+              lconf.soclog,
+              "--output", unseg_path(lconf),
+              stderr=log)
 
 
 def _segment_into_edus(lconf, log):
@@ -126,14 +121,15 @@ def _unit_annotations(lconf, log):
     corpus_dir = minicorpus_path(lconf)
     d_model_path = dact_model_path(lconf, DIALOGUE_ACT_LEARNER)
     d_features_path = dact_features_path(lconf)
+    # 2017-02-16 separate file for labels
+    d_label_path = d_features_path.rsplit('.', 1)[0] + '.labels'  # DIRTY
     d_vocab_path = d_features_path + '.vocab'
-
     lconf.pyt("stac/unit_annotations.py",
               corpus_dir,
               lconf.abspath(LEX_DIR),
               "--model", d_model_path,
               "--vocab", d_vocab_path,
-              "--labels", d_features_path,
+              "--labels", d_label_path,
               "--output", corpus_dir,
               stderr=log)
 
@@ -157,7 +153,7 @@ def _feature_extraction(lconf, log):
     Extract features from our input glozz file
     """
     corpus_dir = minicorpus_path(lconf)
-    vocab_path = lconf.mpack_paths(test_data=False)[3]
+    vocab_path = lconf.mpack_paths(test_data=False)['vocab']
     cmd = ["stac-learning", "extract",
            "--parsing",
            "--vocab", vocab_path,

stac/harness/cmd/stop.py

@@ -6,10 +6,9 @@
 """
 
 from __future__ import print_function
-from attelo.harness.util import call
 import zmq
 
-from ..local import LEX_DIR, CORENLP_ADDRESS
+from ..local import CORENLP_ADDRESS
 
 NAME = 'stop'
 

stac/harness/config/common.py

@@ -6,17 +6,13 @@
 #                                    AstarDecoder,
 #                                    Heuristic,
 #                                    RfcConstraint)
-from attelo.decoding.baseline import (LastBaseline,
-                                      LocalBaseline)
-from attelo.harness.config import (EvaluationConfig,
-                                   LearnerConfig,
-                                   Keyed)
-from attelo.learning.oracle import (AttachOracle, LabelOracle)
+from attelo.decoding.baseline import LastBaseline, LocalBaseline
+from attelo.harness.config import EvaluationConfig, LearnerConfig, Keyed
+from attelo.learning.oracle import AttachOracle, LabelOracle
 from attelo.parser.attach import AttachClassifierWrapper
-from attelo.parser.full import (JointPipeline,
-                                PostlabelPipeline)
-from attelo.parser.label import (LabelClassifierWrapper, SimpleLabeller)
-from attelo.parser.pipeline import (Pipeline)
+from attelo.parser.full import JointPipeline, PostlabelPipeline
+from attelo.parser.label import LabelClassifierWrapper
+from attelo.parser.pipeline import Pipeline
 
 
 def combined_key(*variants):

stac/harness/corenlp.py

@@ -16,7 +16,6 @@
 import sys
 
 import zmq
-import educe.stac
 from educe.stac.corenlp import turn_id_text, parsed_file_name
 
 
@@ -41,9 +40,8 @@ def _launch(config, status):
     to True to indicate that we have launched the server
     (but not necessarily that it is ready to receive anything)
     """
-    subprocess.Popen(["java", 
-                      "-jar",
-                      "target/corenlp-server-0.1.jar",
+    subprocess.Popen(["java",
+                      "-jar", "target/corenlp-server-0.1.jar",
                       "-ssplit.eolonly", "true"],
                      cwd=config.directory,
                      stdout=config.output)

stac/harness/harness.py

@@ -2,6 +2,7 @@
 Paths to files used or generated by the test harness
 '''
 from collections import Counter
+from glob import glob
 from os import path as fp
 import sys
 
@@ -47,8 +48,8 @@ def run(self, runcfg):
             exit_ungathered()
         eval_dir, scratch_dir = prepare_dirs(runcfg, data_dir)
         self.load(runcfg, eval_dir, scratch_dir)
-        evidence_of_gathered = self.mpack_paths(False)[0]
-        if not fp.exists(evidence_of_gathered):
+        evidence_of_gathered = self.mpack_paths(False)['edu_input']
+        if not glob(evidence_of_gathered):
             exit_ungathered()
         evaluate_corpus(self)
 
@@ -116,25 +117,59 @@ def _eval_data_path(self, ext, test_data=False):
 
     def _model_basename(self, rconf, mtype, ext):
         "Basic filename for a model"
-
         if 'attach' in mtype:
             rsubconf = rconf.attach
         else:
             rsubconf = rconf.label
-
         template = '{dataset}.{learner}.{task}.{ext}'
         return template.format(dataset=self.dataset,
                                learner=rsubconf.key,
                                task=mtype,
                                ext=ext)
 
     def mpack_paths(self, test_data, stripped=False):
-        ext = 'relations.sparse'
-        core_path = self._eval_data_path(ext, test_data=test_data)
-        return (core_path + '.edu_input',
-                core_path + '.pairings',
-                (core_path + '.stripped') if stripped else core_path,
-                core_path + '.vocab')
+        """Get paths for the files in the MultiPack.
+
+        Parameters
+        ----------
+        test_data : boolean
+            If True, require paths for the test section of the dataset ;
+            otherwise for the train section.
+
+        stripped : boolean, defaults to False
+            If True, TODO.
+
+        Returns
+        -------
+        file_paths : :obj:`dict` of (str, str or :obj:`repr` list of str)
+            Paths to the files of the MultiPack.
+        """
+        # FIXME catch-up with the new API in educe and attelo
+        # e.g. one group of files per corpus (split) vs per doc or
+        # dialogue
+        base = 'relations'
+        ext = base + '.sparse'
+        # path to data files in the evaluation dir
+        dset = self.testset if test_data else self.dataset
+        # * common files
+        vocab_path = fp.join(self.eval_dir, "%s.%s.vocab" % (dset, ext))
+        labels_path = fp.join(self.eval_dir, "%s.%s.labels" % (dset, base))
+        # * prefix common to data files
+        core_path = fp.join(self.eval_dir, dset, "*.%s" % ext)
+        res = {
+            # data files, might be one per corpus split or one per doc or
+            # dialogue
+            # FIXME values should be lists
+            'edu_input': core_path + '.edu_input',
+            'pairings': core_path + '.pairings',
+            'features': ((core_path + '.stripped') if stripped
+                                 else core_path),
+            # "global" files: vocabulary, labels (unique to a corpus or
+            # corpus split)
+            'vocab': vocab_path,
+            'labels': labels_path
+        }
+        return res
 
     def model_paths(self, rconf, fold, parser):
         """Paths to the learner(s) model(s).