init

MHGTagger/CRFTagger.py

@@ -0,0 +1,110 @@
+
+import sys
+import torch
+from torch import nn
+from torch.nn import functional as F
+
+from .RNNTagger import RNNTagger
+
+
+### auxiliary functions ############################################
+
+def logsumexp(x, dim):
+    """ sums up log-scale values """
+    offset, _ = torch.max(x, dim=dim)
+    offset_broadcasted = offset.unsqueeze(dim) 
+    safe_log_sum_exp = torch.log(torch.exp(x-offset_broadcasted).sum(dim=dim))
+    return safe_log_sum_exp + offset
+
+def lookup(T, indices):
+    """ look up probabilities of tags in a vector, matrix, or 3D tensor """
+    if T.dim() == 3:
+        return T.gather(2, indices.unsqueeze(2)).squeeze(2)
+    elif T.dim() == 2:
+        return T.gather(1, indices.unsqueeze(1)).squeeze(1)
+    elif  T.dim() == 1:
+        return T[indices]
+    else:
+        raise Exception('unexpected tensor size in function "lookup"')
+
+    
+### tagger class ###############################################
+
+class CRFTagger(nn.Module):
+    """ implements a CRF tagger """
+    
+    def __init__(self, num_chars, num_tags, char_emb_size,
+                 char_rec_size, word_rec_size, word_rnn_depth, 
+                 dropout_rate, word_emb_size, beam_size):
+
+        super(CRFTagger, self).__init__()
+
+        # simple LSTMTagger which computes tag scores
+        self.base_tagger = RNNTagger(num_chars, num_tags, char_emb_size,
+                                     char_rec_size, word_rec_size,
+                                     word_rnn_depth, dropout_rate, word_emb_size)
+        self.beam_size = beam_size if 0 < beam_size < num_tags else num_tags
+        self.weights = nn.Parameter(torch.zeros(num_tags, num_tags))
+        self.dropout = nn.Dropout(dropout_rate)
+
+        
+    def forward(self, fwd_charIDs, bwd_charIDs, tags=None):
+
+        annotation_mode = (tags is None)
+
+        scores = self.base_tagger(fwd_charIDs, bwd_charIDs)
+        
+        # extract the highest-scoring tags for each word and their scores
+        best_scores, best_tags = scores.topk(self.beam_size, dim=-1)
+
+        if self.training:  # not done during dev evaluation
+            # check whether the goldstandard tags are among the best tags
+            gs_contained = (best_tags == tags.unsqueeze(1)).sum(dim=-1)
+
+            # replace the tag with the lowest score at each position
+            # by the gs tag if the gs tag is not in the list
+            last_column = gs_contained * best_tags[:,-1] + (1-gs_contained) * tags
+            s = lookup(scores, last_column)
+            best_tags   = torch.cat((best_tags[:,:-1], last_column.unsqueeze(1)), dim=1)
+            best_scores = torch.cat((best_scores[:,:-1], s.unsqueeze(1)), dim=1)
+
+        best_previous = []  # stores the backpointers of the Viterbi algorithm
+        viterbi_scores = best_scores[0]
+        if not annotation_mode:
+            forward_scores = best_scores[0]
+        for i in range(1,scores.size(0)):   # for all word positions except the first
+            # lookup of the tag-pair weights
+            w = self.weights[best_tags[i-1]][:,best_tags[i]]
+            
+            # Viterbi algorithm
+            values = viterbi_scores.unsqueeze(1) + best_scores[i].unsqueeze(0) + w
+            viterbi_scores, best_prev = torch.max(values, dim=0)
+            best_previous.append(best_prev)
+            
+            # Forward algorithm
+            if not annotation_mode:
+                values = forward_scores.unsqueeze(1) + best_scores[i].unsqueeze(0) + w
+                forward_scores = logsumexp(values, dim=0)
+
+        # Viterbi algorithm
+        _, index = torch.max(viterbi_scores, dim=0)
+        best_indices = [index]
+        for i in range(len(best_previous)-1, -1, -1):
+            index = best_previous[i][index]
+            best_indices.append(index)
+
+        # reverse the indices and map them to tag IDs
+        best_indices = torch.stack(best_indices[::-1])
+        predicted_tags = lookup(best_tags, best_indices)
+
+        if annotation_mode:
+            return predicted_tags
+        else:
+            # loss computation
+            basetagger_scores = lookup(scores, tags).sum()
+            CRFweights = self.weights[tags[:-1], tags[1:]].sum() if tags.size(0)>1 else 0
+            logZ = logsumexp(forward_scores, dim=0)  # log partition function
+            logprob = basetagger_scores + CRFweights - logZ
+            
+            return predicted_tags, -logprob
+        

MHGTagger/Data.py

@@ -0,0 +1,186 @@
+
+import sys
+from collections import Counter, OrderedDict 
+import pickle
+import numpy
+
+unk_string   = '<UNK>'
+pad_string   = '<PAD>'
+
+def read_tagged_sentences(path, max_sent_len):
+   """
+   Read a dataset.
+   Each line consists of a token and a tag separated by a tab character
+   """
+   sentences, words, tags = [], [], []
+   with open(path) as file:
+      for line in file:
+         line = line.rstrip()
+         if line:
+            word, tag, *_ = line.split("\t")
+            words.append(word)
+            tags.append(tag)
+         else:
+            # empty line marking the end of a sentence
+            if 0 < len(words) < max_sent_len:
+               sentences.append((words, tags))
+            words, tags = [], []
+   return sentences
+
+
+def read_word_embeddings(filename):
+   # Read word embeddings from file.
+   word_embeddings = []
+   if filename is not None:
+      print("reading word embeddings ...", file=sys.stderr)
+      with open(filename) as file:
+         for line in file:
+            word, *vec = line.rstrip().split(' ')
+            if word != unk_string:
+               word_embeddings.append((word, numpy.array(vec, dtype=numpy.float32)))
+      print("done", file=sys.stderr)
+   word_emb_size = len(word_embeddings[0][1]) if word_embeddings else 0
+   return word_embeddings, word_emb_size
+            
+
+def make_dict(counter, min_freq=0, add_pad_symbol=False):
+   """
+   Create a dictionary which maps strings with some minimal frequency to numbers.
+   We don't use pack_padded sequence, so it is OK to assign ID 1 to the
+   padding symbol.
+   """
+   symlist = [unk_string] + ([pad_string] if add_pad_symbol else []) + \
+             [elem for elem,freq in counter.most_common() if freq>=min_freq]
+   string2ID = {elem:i for i,elem in enumerate(symlist)}
+   return string2ID, symlist
+
+
+class Data(object):
+   """
+   class for reading a tagged training and development corpus or a test corpus
+   """
+   
+   IGNORE_INDEX = -100
+
+   def __init__(self, *args):
+      if len(args) == 1:
+         self.init_test(*args)
+      else:
+         self.init_train(*args)
+
+   ### functions needed during training ###############################################
+
+   def init_train(self, path_train, path_dev, word_trunc_len,
+                  min_char_freq, max_sent_len, word_embeddings, ignore_tag):
+
+      self.word_trunc_len = word_trunc_len  # length to which words are truncated or filled up
+
+      # reading the datasets
+      self.train_sentences = read_tagged_sentences(path_train, max_sent_len)
+      self.dev_sentences   = read_tagged_sentences(path_dev, max_sent_len)
+   
+      ### create dictionaries which map characters or tags to IDs
+      char_counter = Counter()
+      tag_counter  = Counter()
+      for words, tags in self.train_sentences:
+         tag_counter.update(tags)
+         for word in words:
+            char_counter.update(word)
+      self.char2ID, _ = make_dict(char_counter, min_char_freq, add_pad_symbol=True)
+
+      if ignore_tag is not None:
+         tag_counter.pop(ignore_tag, None) # remove this special tag if present
+         self.tag2ID, self.ID2tag  = make_dict(tag_counter)
+         self.tag2ID[ignore_tag] = self.IGNORE_INDEX  # empty tags will not be trained
+      else:
+         self.tag2ID, self.ID2tag  = make_dict(tag_counter)
+
+      ### sizes of the symbol inventories
+      self.num_char_types = len(self.char2ID)
+      self.num_tag_types  = len(self.ID2tag)
+
+      self.word_embeddings, self.word_emb_size = read_word_embeddings(word_embeddings)
+      
+
+   def get_charIDs(self, word):
+      '''
+      maps a word to a sequence of character IDs
+      '''
+
+      unkID = self.char2ID[unk_string]
+      padID = self.char2ID[pad_string]
+
+      charIDs = [self.char2ID.get(c, unkID) for c in word]
+
+      # add enough padding symbols
+      fwd_charIDs = [padID] * self.word_trunc_len + charIDs
+      bwd_charIDs = [padID] * self.word_trunc_len + charIDs[::-1]
+
+      # truncate
+      fwd_charIDs = fwd_charIDs[-self.word_trunc_len:]
+      bwd_charIDs = bwd_charIDs[-self.word_trunc_len:]
+
+      return fwd_charIDs, bwd_charIDs
+
+
+   def words2charIDvec(self, words):
+      """
+      converts words to char-ID vectors
+      """
+
+      ### convert words to character ID sequences
+      fwd_charID_seqs = []
+      bwd_charID_seqs = []
+      for word in words:
+         fwd_charIDs, bwd_charIDs = self.get_charIDs(word)
+         fwd_charID_seqs.append(fwd_charIDs)
+         bwd_charID_seqs.append(bwd_charIDs)
+
+      fwd_charID_seqs = numpy.asarray(fwd_charID_seqs, dtype='int32')
+      bwd_charID_seqs = numpy.asarray(bwd_charID_seqs, dtype='int32')
+
+      return fwd_charID_seqs, bwd_charID_seqs
+
+
+   def tags2IDs(self, tags):
+      """
+      takes a list of tags and converts them to IDs using the tag2ID dictionary
+      """
+      unkID = self.tag2ID[unk_string]
+      IDs = [self.tag2ID.get(tag, unkID) for tag in tags]
+      return numpy.asarray(IDs, dtype='int32')
+
+
+   def save_parameters(self, filename):
+      """ save parameters to a file """
+      all_params = (self.word_trunc_len, self.char2ID, self.ID2tag)
+      with open(filename, "wb") as file:
+         pickle.dump(all_params, file)
+
+
+   ### functions needed during tagging ###############################################
+
+   def init_test(self, filename):
+      """ load parameters from a file """
+      with open(filename, "rb") as file:
+         self.word_trunc_len, self.char2ID, self.ID2tag = pickle.load(file)
+
+   def sentences(self, filename):
+      """ read data to be tagged. One token per line. Empty line follows a sentence """
+      with open(filename) as f:
+         words = []
+         for line in f:
+            line = line.rstrip()
+            if line != '':
+               words.append(line)
+            elif len(words) > 0:
+               # empty line indicates the end of a sentence
+               yield words
+               words = []
+   
+   def single_sentences(self, sentence):
+      yield sentence
+
+   def IDs2tags(self, IDs):
+      """ takes a list of IDs and converts them to tags using the ID2tag dictionary """
+      return [self.ID2tag[int(ID)] for ID in IDs]

MHGTagger/RNNTagger.py

@@ -0,0 +1,111 @@
+
+import sys
+import torch
+from torch import nn
+
+
+class WordRepresentation(nn.Module):
+   '''
+   RNN for computing character-based word representations
+   '''
+   def __init__(self, num_chars, emb_size, rec_size, dropout_rate):
+      super().__init__()
+
+      # character embedding lookup table
+      self.embeddings = nn.Embedding(num_chars, emb_size)
+
+      # character-based LSTMs
+      self.fwd_rnn = nn.LSTM(emb_size, rec_size)
+      self.bwd_rnn = nn.LSTM(emb_size, rec_size)
+
+      self.dropout = nn.Dropout(dropout_rate)
+      
+         
+   def forward(self, fwd_charIDs, bwd_charIDs):
+      # swap the 2 dimensions and lookup the embeddings
+      fwd_embs = self.embeddings(fwd_charIDs.t())
+      bwd_embs = self.embeddings(bwd_charIDs.t())
+
+      # run the biLSTM over characters
+      fwd_outputs, _ = self.fwd_rnn(fwd_embs)
+      bwd_outputs, _ = self.bwd_rnn(bwd_embs)
+
+      # concatenate the forward and backward final states to form
+      # word representations
+      word_reprs = torch.cat((fwd_outputs[-1], bwd_outputs[-1]), -1)
+      
+      return word_reprs
+
+
+class ResidualLSTM(nn.Module):
+   ''' Deep BiRNN with residual connections '''
+   
+   def __init__(self, input_size, rec_size, num_rnns, dropout_rate):
+      super().__init__()
+      self.rnn = nn.LSTM(input_size, rec_size, 
+                         bidirectional=True, batch_first=True)
+
+      self.deep_rnns = nn.ModuleList([
+         nn.LSTM(2*rec_size, rec_size, bidirectional=True, batch_first=True)
+         for _ in range(num_rnns-1)])
+      
+      self.dropout = nn.Dropout(dropout_rate)
+
+   def forward(self, state):
+      state, _ = self.rnn(state)
+      for rnn in self.deep_rnns:
+            hidden, _ = rnn(self.dropout(state))
+            state = state + hidden # residual connection
+      return state
+
+
+class RNNTagger(nn.Module):
+   ''' main tagger module '''
+
+   def __init__(self, num_chars, num_tags, char_emb_size, char_rec_size, 
+                word_rec_size, word_rnn_depth, dropout_rate, word_emb_size):
+
+      super().__init__()
+
+      # character-based BiLSTMs
+      self.word_representations = WordRepresentation(num_chars, char_emb_size, 
+                                                     char_rec_size, dropout_rate)
+      # word-based BiLSTM
+      self.word_rnn = ResidualLSTM(char_rec_size*2, word_rec_size, word_rnn_depth,
+                                   dropout_rate)
+      # output feed-forward network
+      self.output_layer = nn.Linear(2*word_rec_size, num_tags)
+
+      # dropout layers
+      self.dropout = nn.Dropout(dropout_rate)
+
+      # word embedding projection layer for finetuning on word embeddings
+      if word_emb_size > 0:
+         self.projection_layer = nn.Linear(2*char_rec_size, word_emb_size)
+
+
+   def forward(self, fwd_charIDs, bwd_charIDs, word_embedding_training=False):
+         
+      # compute the character-based word representations
+      word_reprs = self.word_representations(fwd_charIDs, bwd_charIDs)
+
+      if word_embedding_training:
+         if not hasattr(self, 'projection_layer'):
+            sys.exit("Error: The embedding projection layer is undefined!")
+         # Project the word representations to word embedding vectors
+         # for finetuning on word embeddings as an auxiliary task
+         word_embs = self.projection_layer(word_reprs)
+         return word_embs
+
+      # apply dropout
+      word_reprs = self.dropout(word_reprs)
+         
+      # run the BiLSTM over words
+      reprs = self.word_rnn(word_reprs.unsqueeze(0)).squeeze(0)
+      reprs = self.dropout(reprs)  # and apply dropout
+      
+      # apply the output layers
+      scores = self.output_layer(reprs)
+      
+      return scores
+      

MHGTagger/rnn_annotate.py

@@ -0,0 +1,145 @@
+#!/usr/bin/python3
+
+import sys
+import pickle
+import torch
+from huggingface_hub import hf_hub_download
+
+from .Data import Data
+from .RNNTagger import RNNTagger
+from .CRFTagger import CRFTagger
+
+
+###########################################################################
+# main function
+###########################################################################
+
+class Args:
+   def __init__(self, path_param, model_id, path_data, crf_beam_size, gpu, min_prob, print_probs) -> None:
+      self.path_param = path_param
+      self.model_id = model_id
+      self.path_data = path_data
+      self.crf_beam_size = crf_beam_size
+      self.gpu = gpu
+      self.min_prob = min_prob
+      self.print_probs = print_probs
+
+# if __name__ == "__main__":
+def annotate(tokens, path_param='MHGTagger/tagger', model_id='nielklug/rnn_tagger', path_data='', crf_beam_size=10, gpu=-1, min_prob=-1.0, print_probs=True):
+
+   # parser = argparse.ArgumentParser(description='Annotation program of the RNN-Tagger.')
+
+   # parser.add_argument('path_param', type=str,
+   #                     help='name of parameter file')
+   # parser.add_argument('path_data', type=str,
+   #                     help='name of the file with input data')
+   # parser.add_argument('--crf_beam_size', type=int, default=10,
+   #                     help='size of the CRF beam (if the system contains a CRF layer)')
+   # parser.add_argument('--gpu', type=int, default=0,
+   #                     help='selection of the GPU. The default is: 0 (CPU=-1)')
+   # parser.add_argument("--min_prob", type=float, default=-1.0,
+   #                     help="print all tags whose probability exceeds the probability of the best tag times this threshold")
+   # parser.add_argument("--print_probs", action="store_true", default=False,
+   #                     help="print the tag probabilities")
+
+   args = Args(path_param, model_id, path_data, crf_beam_size, gpu, min_prob, print_probs)
+
+   # Select the processing device
+   if args.gpu >= 0:
+      if not torch.cuda.is_available():
+         print('No gpu available. Using cpu instead.', file=sys.stderr)
+         args.gpu = -1
+      else:
+         if args.gpu >= torch.cuda.device_count():
+            print('gpu '+str(args.gpu)+' not available. Using gpu 0 instead.', file=sys.stderr)
+            args.gpu = 0
+         torch.cuda.set_device(args.gpu)
+   device = torch.device('cuda' if args.gpu >= 0 else 'cpu')
+
+   # load parameters
+   data  = Data(args.path_param+'.io')   # read the symbol mapping tables
+
+   with open(args.path_param+'.hyper', 'rb') as file:
+      hyper_params = pickle.load(file)
+   model = CRFTagger(*hyper_params) if len(hyper_params)==10 \
+           else RNNTagger(*hyper_params)
+   
+   model_file = hf_hub_download(repo_id=args.model_id, filename='tagger.rnn')
+   model.load_state_dict(torch.load(model_file, 
+                         map_location=torch.device('cpu')))
+
+   model = model.to(device)
+
+   if type(model) is CRFTagger:
+      for optvar, option in zip((args.min_prob, args.print_probs),
+                                ("min_prob","print_probs")):
+         if optvar:
+            print(f"Warning: Option --{option} is ignored because the model has a CRF output layer", file=sys.stderr)
+   
+   model.eval()
+   with torch.no_grad():
+      for i, words in enumerate(data.single_sentences(tokens)):
+         # print(i, end='\r', file=sys.stderr, flush=True)
+   
+         # map words to numbers and create Torch variables
+         fwd_charIDs, bwd_charIDs = data.words2charIDvec(words)
+         fwd_charIDs = torch.LongTensor(fwd_charIDs).to(device)
+         bwd_charIDs = torch.LongTensor(bwd_charIDs).to(device)
+         
+         words_all = []
+         tagged = []
+         probs_all = []
+         # run the model
+         if type(model) is RNNTagger:
+            tagscores = model(fwd_charIDs, bwd_charIDs)
+            if args.min_prob == -1.0:
+               # only print the word and tag with the highest score
+               tagIDs = tagscores.argmax(-1)
+               tags = data.IDs2tags(tagIDs.to("cpu"))
+               if not args.print_probs:
+                  for word, tag in zip(words, tags):
+                     # print(word, tag, sep="\t")
+                     words_all.append(word)
+                     tagged.append(tag)
+               else:
+                  # print probabilities as well
+                  tagprobs = torch.nn.functional.softmax(tagscores, dim=-1)
+                  # get the probabilities of the highest-scoring tags
+                  probs = tagprobs[range(len(tagIDs)), tagIDs].to("cpu").tolist()
+                  # print the result
+                  for word, tag, prob in zip(words, tags, probs):
+                     # print(word, tag, round(float(prob), 4), sep="\t")
+                     words_all.append(word)
+                     tagged.append(tag)
+                     probs_all.append(round(float(prob), 4))
+            else:
+               # print the best tags for each word
+               tagprobs = torch.nn.functional.softmax(tagscores, dim=-1)
+               # get the most probable tag and its probability
+               best_probs, _ = tagprobs.max(-1)
+               # get all tags with a probability above best_prob * min_prob
+               thresholds = best_probs * args.min_prob
+               greaterflags = (tagprobs > thresholds.unsqueeze(1))
+               for word, flags, probs in zip(words, greaterflags, tagprobs):
+                  # get the IDs of the best tags
+                  IDs = flags.nonzero()
+                  # get the best tags and their probabilities
+                  best_probs = probs[IDs].to("cpu")
+                  best_tags = data.IDs2tags(IDs.to("cpu"))
+                  # sort the tags by decreasing probability
+                  sorted_list = sorted(zip(best_tags, best_probs), key=lambda x:-x[1])
+                  best_tags, best_probs = zip(*sorted_list)
+                  # generate the output
+                  if args.print_probs:
+                     # append the probabilities to the tags
+                     best_tags = [f"{t} {float(p):.4f}" for t, p in zip(best_tags, best_probs)]
+                  print(word, ' '.join(best_tags), sep="\t")
+         elif type(model) is CRFTagger:
+            tagIDs = model(fwd_charIDs, bwd_charIDs)
+            tags = data.IDs2tags(tagIDs)
+            for word, tag in zip(words, tags):
+               print(word, tag, sep='\t')
+         else:
+            sys.exit('Error')
+   
+         return (words_all, tagged, probs_all)

README.md

@@ -1,5 +1,5 @@
 ---
-title: Mhg Parsing
+title: MHG Parsing
 emoji: 🌍
 colorFrom: gray
 colorTo: red

Tagset_Mappings/POS-mapping.txt

@@ -0,0 +1,73 @@
+$_	$_
+ADJA	ADJA
+ADJD	ADJD
+ADJN	ADJA.Pos
+ADJS	ADJA
+APPR	APPR
+APPRART	APPRART
+AVD	ADV
+AVD-KO*	ADV
+AVG	PWAV
+AVW	PWAV
+CARDA	CARD
+CARDD	CARD
+CARDN	CARD
+CARDS	CARD
+DDA	PDAT
+DDART	ART
+DDD	PDAT
+DDN	PDAT
+DDS	PDS
+DGA	PWAT
+DGS	PWS
+DIA	PIAT
+DIART	ART
+DID	PDAT
+DIN	PDAT
+DIS	PIS
+DPOSA	PPOSAT
+DPOSD	PPOSS
+DPOSN	PPOSAT
+DPOSS	NN
+DRELS	PRELS
+DWA	PWAT
+DWD	PWS
+DWS	PWS
+FM	FM
+ITJ	ITJ
+KO*	KOUS
+KOKOM	KOKOM
+KON	KON
+KOUS	KOUS
+NA	NN
+NE	NE
+PART	PART
+PAVAP	PROAV
+PAVD	PROAV
+PAVG	PROAV
+PAVW	PWAV
+PG	PWS
+PI	PIS
+PPER	PPER
+PRF	PRF
+PTK	ADV
+PTKA	PTKA
+PTKANT	PTKANT
+PTKNEG	PTKNEG
+PTKVZ	PTKVZ
+PW	PWS
+VAFIN	VAFIN
+VAIMP	VAIMP
+VAINF	VAINF
+VAPP	VAPP
+VAPS	ADJD.Pos
+VMFIN	VMFIN
+VMIMP	VMIMP
+VMINF	VMINF
+VMPP	VMPP
+VMPS	ADJD.Pos
+VVFIN	VVFIN
+VVIMP	VVIMP
+VVINF	VVINF
+VVPP	VVPP
+VVPS	ADJD.Pos

Tagset_Mappings/feature-mapping.txt

@@ -0,0 +1,11 @@
+Masc,Fem	*
+Fem,Masc	*
+Masc,Neut	*
+Neut,Masc	*
+Fem,Neut	*
+Neut,Fem	*
+Abl	Dat
+Instr	Dat
+Akk	Acc
+Voc	Nom
+bSg	Sg

Tagset_Mappings/tag_mapping.py

@@ -0,0 +1,129 @@
+#!/usr/bin/python3
+
+"""
+cd schmid/MHG-Parser/Tagset-Mappings
+python tag-mapping.py ../self-attentive-parser-master/data/mhg/MHG.tagged > ../self-attentive-parser-master/data/mhg/MHG_new.mapped
+"""
+
+import sys
+import fileinput
+
+with open("Tagset_Mappings/POS-mapping.txt") as file:
+    pos_map = dict(line.split() for line in file if line.strip())
+
+with open("Tagset_Mappings/feature-mapping.txt") as file:
+    feature_map = dict(line.split() for line in file if line.strip())
+
+def map_tags(tags):
+    return [map_tag(tag) for tag in tags]
+        
+
+def map_tag(tag):
+    tag.replace('AVD.Comp', 'AVD').replace('AVD.Sup', 'AVD')
+    pos, *features = tag.split(".")
+    pos = pos.split('|')[0]
+    pos = pos_map[pos]
+    pos, *features2 = pos.split(".")
+    features = features2 + features
+    features = [feature_map.get(f, f) for f in features]
+    if pos == 'ADJA':
+        if len(features) == 5:
+            features = [features[0], features[2], features[3], features[1]]
+        elif len(features) in [3,4]:
+            features = [features[0], features[2], '*', features[1]]
+        elif len(features) == 2:
+            features = [features[0], '*', '*', features[1]]
+        elif len(features) == 1:
+            features = [features[0], '*', '*', '*']
+    elif pos in ['ADV', 'CARD']:
+        features = []
+    elif pos in ['ART', 'APPRART']:
+        if len(features) == 4:
+            features = [features[1], features[2], features[0]]
+        elif len(features) in [0, 1]:
+            features = ['*', '*', '*']
+    elif pos == 'NN':
+        if len(features) == 4:
+            features = [features[1], features[2], features[0]]
+        elif len(features) == 0:
+            features = ['*', '*', '*']
+    elif pos == 'NE':
+        if len(features) == 2:
+            features.append('*')
+        elif len(features) == 1:
+            features.extend(['*', '*'])
+    elif pos == 'PDAT':
+        if len(features) == 4:
+            features = [features[1], features[2], features[0]]
+        elif len(features) == 0:
+            features = ['*', '*', '*']
+    elif pos == 'PIAT':
+        if len(features) == 4:
+            features = [features[1], features[2], features[0]]
+        if len(features) == 2:
+            features = [features[1], '*', features[0]]
+        elif len(features) == 0:
+            features = ['*', '*', '*']
+    elif pos == 'PPOSAT':
+        if len(features) in [3, 4]:
+            features = [features[1], features[2], features[0]]
+        elif len(features) == 0:
+            features = ['*', '*', '*']
+    elif pos == 'PWAT' and len(features) == 4:
+        features = [features[1], features[2], features[0]]
+    elif pos == 'PPOSS':
+        features = ['*.*.*']
+    elif pos == 'PDS':
+        if len(features) == 4:
+            features = [features[1], features[2], features[0]]
+        elif len(features) == 1:
+            features.extend(['*', '*'])
+        elif len(features) == 2:
+            features = [features[1], '*', '*']
+    elif pos == 'PIS':
+        if len(features) == 4:
+            features = [features[1], features[2], features[0]]
+        elif len(features) == 0:
+            features = ['*', '*', '*']
+    elif pos == 'PWS':
+        if len(features) == 4:
+            features = [features[1], features[2], features[0]]
+        elif len(features) == 0:
+            features = ['*', '*', '*']
+    elif pos == 'PRELS' and len(features) == 3:
+        features = [features[1], features[2], features[0]]
+    elif pos == 'PPER' and len(features) == 4:
+        features = [features[3], features[1], features[2], features[0]]
+    elif pos == 'PRF' and len(features) == 3:
+        features = ['*', features[0], features[1]]
+    elif pos in ['VAFIN','VMFIN','VVFIN'] and len(features) == 4:
+        features = [features[3], features[2], features[1], features[0]]
+    elif pos in ['VAIMP','VMIMP','VVIMP'] and len(features) == 2:
+        features = [features[1], features[0], 'Imp']
+    elif pos in ['VAINF','VMINF','VVINF'] and len(features) == 0:
+        features = ['Inf']
+    elif pos in ['VAPP','VMPP','VVPP'] and len(features) == 0:
+        features = ['Psp']
+    return '.'.join([pos]+features)
+    
+# for i, line in enumerate(fileinput.input()):
+#     print(i, end="\r", file=sys.stderr)
+#     line = line.strip()
+#     if line:
+#         word, tag, *_ = line.split("\t")
+#         tag = tag.replace('APPR|DDART', 'APPRART')
+#         for t in tag.split("|"):
+#             print(word, map_tag(t), sep="\t")
+#     else:
+#         print()
+
+
+# for i, line in enumerate(fileinput.input()):
+#     print(i, end="\r", file=sys.stderr)
+#     line = line.strip()
+#     if line:
+#         word, tag, *_ = line.split("\t")
+#         tag = tag.replace('APPR|DDART', 'APPRART')
+#         print(word, map_tag(tag.split('|')[0]), sep="\t")
+#     else:
+#         print()

app.py

@@ -0,0 +1,47 @@
+import streamlit as st
+from parse import parse_text
+import nltk
+from nltk import Tree
+import pandas as pd
+import re
+from nltk.tree.prettyprinter import TreePrettyPrinter
+
+
+st.title("MHG parsing system (demo)")
+text = st.text_area("""This is a simple demo of a Middle High German (MHG) parsing system using delexicalization method.\n\n
+                    Enter some MHG text below!""")
+
+st.text("""Example MHG sentences:
+1. Swer an rehte güete wendet sîn gemüete, dem volget sælde und êre, des gît gewisse 
+lêre künec Artûs der guote, der mit rîters muote nâch lobe kunde strîten.
+2. Uns ist in alten mæren wunders vil geseitvon helden lobebæren, von grôzer arebeit,
+von freuden, hôchgezîten, von weinen und von klagen, von küener recken strîten muget 
+ir nu wunder hœren sagen.""")
+
+nltk.download('punkt')
+
+
+if text:
+  tokens, tags, probs, parse_tree = parse_text(text)
+  
+  # create a table to show the tagged results:
+  zipped = list(zip(tokens, tags, probs))
+  
+  df = pd.DataFrame(zipped, columns=['Token', 'Tag', 'Prob.'])
+  
+  # Convert the bracket parse tree into an NLTK Tree
+  t = Tree.fromstring(re.sub(r'(\.[^ )]+)+', '', parse_tree))
+  
+  tree_svg = TreePrettyPrinter(t).svg(nodecolor='black', leafcolor='black', funccolor='black')
+  
+  col1 = st.columns(1)[0]
+  col1.header("POS tagging result:")
+  col1.table(df)
+  
+  col2 = st.columns(1)[0]
+  col2.header("Parsing result:")
+  col2.write(parse_tree.replace('_', '\_').replace('$', '\$').replace('*', '\*'))
+
+# Display the graph in the Streamlit app
+  col2.image(tree_svg, use_column_width=True)
+    

parse.py

@@ -0,0 +1,19 @@
+import re
+from MHGTagger.rnn_annotate import annotate
+from Tagset_Mappings.tag_mapping import map_tags
+from parsing.src.parse import run_parse 
+from nltk import word_tokenize
+
+def parse_text(text):
+    tokens = tokenize(text)
+    tokens, tags, probs = annotate(tokens)
+    tags = map_tags(tags)
+    parse_tree = run_parse(tokens, tags)[0]
+    return tokens, tags, probs, parse_tree
+
+def tokenize(text: str):
+    text = re.sub(r'\s*([.,;:?!"])\s', r' \1 ', text)
+    text = re.sub(r'\s*([.,;:?!"]) ', r' \1 ', text)
+    tokens = word_tokenize(text)
+    return tokens
+    

parsing/EVALB/COLLINS.prm

@@ -0,0 +1,66 @@
+##------------------------------------------##
+## Debug mode                               ##
+##   0: No debugging                        ##
+##   1: print data for individual sentence  ##
+##------------------------------------------##
+DEBUG 0
+
+##------------------------------------------##
+## MAX error                                ##
+##    Number of error to stop the process.  ##
+##    This is useful if there could be      ##
+##    tokanization error.                   ##
+##    The process will stop when this number##
+##    of errors are accumulated.            ##
+##------------------------------------------##
+MAX_ERROR 10
+
+##------------------------------------------##
+## Cut-off length for statistics            ##
+##    At the end of evaluation, the         ##
+##    statistics for the senetnces of length##
+##    less than or equal to this number will##
+##    be shown, on top of the statistics    ##
+##    for all the sentences                 ##
+##------------------------------------------##
+CUTOFF_LEN 40
+
+##------------------------------------------##
+## unlabeled or labeled bracketing          ##
+##    0: unlabeled bracketing               ##
+##    1: labeled bracketing                 ##
+##------------------------------------------##
+LABELED 1                 
+
+##------------------------------------------##
+## Delete labels                            ##
+##    list of labels to be ignored.         ##
+##    If it is a pre-terminal label, delete ##
+##    the word along with the brackets.     ##
+##    If it is a non-terminal label, just   ##
+##    delete the brackets (don't delete     ##
+##    deildrens).                           ##
+##------------------------------------------##
+DELETE_LABEL TOP
+DELETE_LABEL -NONE-
+DELETE_LABEL ,
+DELETE_LABEL :
+DELETE_LABEL ``
+DELETE_LABEL ''
+DELETE_LABEL .
+
+##------------------------------------------##
+## Delete labels for length calculation     ##
+##    list of labels to be ignored for      ##
+##    length calculation purpose            ##
+##------------------------------------------##
+DELETE_LABEL_FOR_LENGTH -NONE-
+
+##------------------------------------------##
+## Equivalent labels, words                 ##
+##     the pairs are considered equivalent  ##
+##     This is non-directional.             ##
+##------------------------------------------##
+EQ_LABEL ADVP PRT
+
+# EQ_WORD  Example example

parsing/EVALB/LICENSE

@@ -0,0 +1,24 @@
+This is free and unencumbered software released into the public domain.
+
+Anyone is free to copy, modify, publish, use, compile, sell, or
+distribute this software, either in source code form or as a compiled
+binary, for any purpose, commercial or non-commercial, and by any
+means.
+
+In jurisdictions that recognize copyright laws, the author or authors
+of this software dedicate any and all copyright interest in the
+software to the public domain. We make this dedication for the benefit
+of the public at large and to the detriment of our heirs and
+successors. We intend this dedication to be an overt act of
+relinquishment in perpetuity of all present and future rights to this
+software under copyright law.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR
+OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+OTHER DEALINGS IN THE SOFTWARE.
+
+For more information, please refer to <http://unlicense.org/>

parsing/EVALB/Makefile

@@ -0,0 +1,4 @@
+all: evalb
+
+evalb: evalb.c
+	gcc -Wall -g -o evalb evalb.c

parsing/EVALB/README

@@ -0,0 +1,300 @@
+#################################################################
+#                                                               #
+#      Bug fix and additional functionality for evalb           #
+#                                                               #
+# This updated version of evalb fixes a bug in which sentences  #
+# were incorrectly categorized as "length mismatch" when the    #
+# the parse output had certain mislabeled parts-of-speech.      #
+#                                                               #
+# The bug was the result of evalb treating one of the tags (in  #
+# gold or test) as a label to be deleted (see sections [6],[7]  #
+# for details), but not the corresponding tag in the other.     #
+# This most often occurs with punctuation. See the subdir       #
+# "bug" for an example gld and tst file demonstating the bug,   #
+# as well as output of evalb with and without the bug fix.      #
+#                                                               #
+# For the present version in case of length mismatch, the nodes #
+# causing the imbalance are reinserted to resolve the miscount. #
+# If the lengths of gold and test truly differ, the error is    #
+# still reported. The parameter file "new.prm" (derived from    #
+# COLLINS.prm) shows how to add new potential mislabelings for  #
+# quotes (",``,',`).                                            #
+#                                                               #
+# I have preserved DJB's revision for modern compilers except   #
+# for the delcaration of "exit" which is provided by stdlib.    #
+#                                                               #
+# Other changes:                                                #
+#                                                               #
+# * output of F-Measure in addition to precision and recall     #
+#   (I did not update the documention in section [4] for this)  #
+#                                                               #
+# * more comprehensive DEBUG output that includes bracketing    #
+#   information as evalb is processing each sentence            #
+#   (useful in working through this, and peraps other bugs).    #
+#   Use either the "-D" run-time switch or set DEBUG to 2 in    #
+#   the parameter file.                                         #
+#                                                               #
+# * added DELETE_LABEL lines in new.prm for S1 nodes produced   #
+#   by the Charniak parser and "?", "!" punctuation produced by #
+#   the Bikel parser.                                           #
+#                                                               #
+#                                                               #
+#                                           David Ellis (Brown) #
+#                                                               #
+#                                           January.2006        #
+#################################################################
+
+#################################################################
+#                                                               #
+#      Update of evalb for modern compilers                     #
+#                                                               #
+# This is an updated version of evalb, for use with modern C    #
+# compilers. There are a few updates, each marked in the code:  #
+#                                                               #
+# /* DJB: explanation of comment */                             #
+#                                                               #
+# The updates are purely to help compilation with recent        #
+# versions of GCC (and other C compilers). There are *NO* other #
+# changes to the algorithm itself.                              #
+#                                                               #
+# I have made these changes following recommendations from      #
+# users of the Corpora Mailing List, especially Peet Morris and #
+# Ramon Ziai.                                                   #
+#                                                               #
+#                                     David Brooks (Birmingham) #
+#                                                               #
+#                                     September.2005            #
+#################################################################
+
+#################################################################
+#                                                               #
+#      README file for evalb                                    #
+#                                                               #
+#                                         Satoshi Sekine (NYU)  #
+#                                         Mike Collins (UPenn)  #
+#                                                               #
+#                                         October.1997          #
+#################################################################
+
+Contents of this README:
+
+   [0] COPYRIGHT
+   [1] INTRODUCTION
+   [2] INSTALLATION AND RUN
+   [3] OPTIONS
+   [4] OUTPUT FORMAT FROM THE SCORER
+   [5] HOW TO CREATE A GOLDFILE FROM THE TREEBANK
+   [6] THE PARAMETER FILE
+   [7] MORE DETAILS ABOUT THE SCORING ALGORITHM
+
+
+[0] COPYRIGHT
+
+The authors abandon the copyright of this program. Everyone is 
+permitted to copy and distribute the program or a portion of the program
+with no charge and no restrictions unless it is harmful to someone.
+
+However, the authors are delightful for the user's kindness of proper
+usage and letting the authors know bugs or problems.
+
+This software is provided "AS IS", and the authors make no warranties,
+express or implied.
+
+To legally enforce the abandonment of copyright, this package is released
+under the Unlicense (see LICENSE).
+
+[1] INTRODUCTION
+
+Evaluation of bracketing looks simple, but in fact, there are minor
+differences from system to system. This is a program to parametarize
+such minor differences and to give an informative result.
+
+"evalb" evaluates bracketing accuracy in a test-file against a gold-file.
+It returns recall, precision, tagging accuracy. It uses an identical 
+algorithm to that used in (Collins ACL97).
+
+
+[2] Installation and Run
+
+To compile the scorer, type 
+
+> make
+
+
+To run the scorer:
+
+> evalb -p Parameter_file Gold_file Test_file
+
+ 
+For example to use the sample files:
+
+> evalb -p sample.prm sample.gld sample.tst
+
+
+
+[3] OPTIONS
+
+You can specify system parameters in the command line options.
+Other options concerning to evaluation metrix should be specified
+in parameter file, described later.
+
+        -p param_file  parameter file                        
+        -d             debug mode                            
+        -e n           number of error to kill (default=10)  
+        -h             help                                  
+
+
+
+[4] OUTPUT FORMAT FROM THE SCORER
+
+The scorer gives individual scores for each sentence, for
+example:
+
+  Sent.                        Matched  Bracket   Cross        Correct Tag
+ ID  Len.  Stat. Recal  Prec.  Bracket gold test Bracket Words  Tags Accracy
+============================================================================
+   1    8    0  100.00 100.00     5      5    5      0      6     5    83.33
+
+At the end of the output the === Summary === section gives statistics 
+for all sentences, and for sentences <=40 words in length. The summary
+contains the following information:
+
+i)   Number of sentences -- total number of sentences.
+
+ii)  Number of Error/Skip sentences -- should both be 0 if there is no
+    problem with the parsed/gold files.
+
+iii) Number of valid sentences = Number of sentences - Number of Error/Skip
+    sentences 
+
+iv)  Bracketing recall =     (number of correct constituents)
+                         ----------------------------------------
+                         (number of constituents in the goldfile)
+
+v)   Bracketing precision = (number of correct constituents)
+                         ----------------------------------------
+                         (number of constituents in the parsed file)
+
+vi)  Complete match = percentaage of sentences where recall and precision are
+    both 100%. 
+
+vii) Average crossing = (number of constituents crossing a goldfile constituen
+                         ----------------------------------------------------
+                                        (number of sentences)
+
+viii) No crossing = percentage of sentences which have 0 crossing brackets.
+
+ix)   2 or less crossing = percentage of sentences which have <=2 crossing brackets.
+
+x)    Tagging accuracy = percentage of correct POS tags (but see [5].3 for exact
+     details of what is counted).
+
+
+
+[5] HOW TO CREATE A GOLDFILE FROM THE PENN TREEBANK
+
+
+The gold and parsed files are in a format similar to this:
+
+(TOP (S (INTJ (RB No)) (, ,) (NP (PRP it)) (VP (VBD was) (RB n't) (NP (NNP Black) (NNP Monday))) (. .)))
+
+To create a gold file from the treebank:
+
+tgrep -wn '/.*/' | tgrep_proc.prl 
+
+will produce a goldfile in the required format.  ("tgrep -wn '/.*/'" prints
+parse trees, "tgrep_process.prl" just skips blank lines).
+
+For example, to produce a goldfile for section 23 of the treebank:
+
+tgrep -wn '/.*/' | tail +90895 | tgrep_process.prl | sed 2416q > sec23.gold
+
+
+
+[6] THE PARAMETER (.prm) FILE
+
+
+The .prm file sets options regarding the scoring method. COLLINS.prm gives
+the same scoring behaviour as the scorer used in (Collins 97). The options 
+chosen were: 
+
+1) LABELED 1
+
+to give labelled precision/recall figures, i.e. a constituent must have the
+same span *and* label as a constituent in the goldfile.
+
+2) DELETE_LABEL TOP   
+
+Don't count the "TOP" label (which is always given in the output of tgrep) 
+when scoring. 
+
+3) DELETE_LABEL -NONE-  
+
+Remove traces (and all constituents which dominate nothing but traces) when
+scoring. For example
+
+.... (VP (VBD reported) (SBAR (-NONE- 0) (S (-NONE- *T*-1)))) (. .)))
+
+would be processed to give
+
+.... (VP (VBD reported)) (. .)))
+
+
+4)
+DELETE_LABEL ,     -- for the purposes of scoring remove punctuation
+DELETE_LABEL :
+DELETE_LABEL ``
+DELETE_LABEL ''
+DELETE_LABEL .
+
+5) DELETE_LABEL_FOR_LENGTH -NONE-   -- don't include traces when calculating
+                                       the length of a sentence (important
+                                       when classifying a sentence as <=40
+                                       words or >40 words)
+
+6) EQ_LABEL ADVP PRT
+
+Count ADVP and PRT as being the same label when scoring.
+
+
+
+
+[7] MORE DETAILS ABOUT THE SCORING ALGORITHM
+
+
+1) The scorer initially processes the files to remove all nodes specified
+by DELETE_LABEL in the .prm file. It also recursively removes nodes which
+dominate nothing due to all their children being removed. For example, if
+-NONE- is specified as a label to be deleted, 
+
+.... (VP (VBD reported) (SBAR (-NONE- 0) (S (-NONE- *T*-1)))) (. .)))
+
+would be processed to give
+
+.... (VP (VBD reported)) (. .)))
+
+2) The scorer also removes all functional tags attached to non-terminals
+(functional tags are prefixed with "-" or "=" in the treebank). For example
+"NP-SBJ" is processed to give "NP", "NP=2" is changed to "NP".
+
+
+3) Tagging accuracy counts tags for all words *except* any tags which are
+deleted by a DELETE_LABEL specification in the .prm file. (For example, for
+COLLINS.prm, punctuation tagged as "," ":" etc. would not be included).
+
+4) When calculating the length of a sentence, all words with POS tags not 
+included in the "DELETE_LABEL_FOR_LENGTH" list in the .prm file are
+counted. (For COLLINS.prm, only "-NONE-" is specified in this list, so
+traces are removed before calculating the length of the sentence).
+
+5) There are some subtleties in scoring when either the goldfile or parsed
+file contains multiple constituents for the same span which have the same
+non-terminal label. e.g. (NP (NP the man)) If the goldfile contains n 
+constituents for the same span, and the parsed file contains m constituents
+with that nonterminal, the scorer works as follows:
+
+i) If m>n, then the precision is n/m, recall is 100%
+
+ii) If n>m, then the precision is 100%, recall is m/n.
+
+iii) If n==m, recall and precision are both 100%.

parsing/EVALB/bug/bug.gld

@@ -0,0 +1,5 @@
+(TOP (S (NP-SBJ (DT The)  (NN Thy-1)  (NN gene)  (NN promoter) ) (VP (VBZ resembles)  (NP (DT a)  (`` ")  (JJ housekeeping)  ('' ")  (NN promoter) ) (PP (IN in)  (SBAR (IN that)  (S (NP-SBJ-68 (PRP it) ) (VP-COOD (VP (VBZ is)  (ADJP-PRD (JJ located)  (PP (IN within)  (NP (DT a)  (JJ methylation-free)  (NN island) )))) (, ,)  (VP (VBZ lacks)  (NP (DT a)  (JJ canonical)  (NN TATA)  (NN box) )) (, ,)  (CC and)  (VP (VBZ displays)  (NP (NN heterogeneity) ) (PP (IN in)  (NP (NP (DT the)  (JJ 5'-end)  (NNS termini) ) (PP (IN of)  (NP (DT the)  (NN mRNA) )))))))))) (. .) ) )
+(TOP (S (NP-SBJ (DT The)  (JJ latter)  (`` ")  (NP (NP (JJ nuclear)  (NN factor) ) (PP (IN for)  (NP (VBN activated)  (NN T)  (NNS cells) ))) ('' ") ) (ADVP (RB likely) ) (VP (VBZ contributes)  (PP (TO to)  (NP (NP (DT the)  (NN tissue)  (NN specificity) ) (PP (IN of)  (NP (NN IL-2)  (NN gene)  (NN expression) ))))) (. .) ) )
+(TOP (S (ADVP (RB Thus) ) (, ,)  (NP-SBJ (PRP we) ) (VP (VBD postulated)  (SBAR-COOD (SBAR (IN that)  (S (NP-SBJ (NP (DT the)  (JJ circadian)  (NN modification) ) (PP (IN of)  (NP (NN GR) ))) (VP (VBD was)  (ADJP-PRD (JJ independent)  (PP (IN of)  (NP-COOD (NP (NP (DT the)  (JJ diurnal)  (NNS fluctuations) ) (PP (IN in)  (NP (NN plasma)  (NN cortisol)  (NN level) ))) (CC or)  (NP (NP (DT the)  (JJ circadian)  (NNS variations) ) (PP (IN in)  (NP (JJ environmental)  (NN lighting) ))))))))) (CC and)  (SBAR (IN that)  (S (NP-SBJ-79 (DT the)  (NN rhythmicity) ) (VP (MD might)  (VP (VB be)  (VP (VBN regulated)  (NP (-NONE- *-79) ) (PP (IN by)  (NP-LGS (NP (DT the)  (`` ')  (JJ circadian)  (NN pacemaker)  ('' ') ) (ADJP (JJ located)  (PP (IN in)  (NP (DT the)  (JJ human)  (JJ basal)  (NN brain) )))))))))))) (. .) ) )
+(TOP (S (NP-SBJ-70 (JJ Such)  (NN transcription)  (NNS factors) ) (VP (VBP play)  (NP (DT a)  (JJ key)  (NN role) ) (PP (IN in)  (NP (NP (DT the)  (NN development) ) (PP (IN of)  (NP (DT the)  (JJ mature)  (NN T-cell)  (NN phenotype) )))) (PP (IN by)  (S (NP-SBJ (-NONE- *-70) ) (VP (VBG functioning)  (PP (IN as)  (`` ')  (NP (NP (JJ master)  (NNS regulators) ) (PP (IN of)  (NP (NN T-cell)  (NN differentiation) ))) ('' ') ))))) (. .) ) )
+(TOP (S (NP-SBJ (NP (DT The)  (NN conversion) ) (PP (IN of)  (NP (DT the)  (NN TCEd) )) (PP (TO to)  (NP (DT a)  (`` ')  (JJ perfect)  ('' ')  (NN NF-kB)  (NN binding)  (NN site) ))) (VP-COOD (VP (VBZ leads)  (PP (TO to)  (NP-19 (NP (DT a)  (JJR tighter)  (NN binding) ) (PP (IN of)  (NP (NN NF-kB) )) (PP (TO to)  (NP (NN TCEd)  (NN DNA) ))))) (CC and)  (, ,)  (VP (PP (IN as)  (NP (DT a)  (JJ functional)  (NN consequence) )) (, ,)  (PP (TO to)  (NP=19 (NP (DT the)  (NN activity) ) (PP (IN of)  (NP (DT the)  (`` ')  (VBN converted)  ('' ')  (NN TCEd)  (NNS motifs) )) (PP (IN in)  (NP (NN HeLa)  (NNS cells) )))))) (. .) ) )

parsing/EVALB/bug/bug.rsl-new

@@ -0,0 +1,39 @@
+Sent.                        Matched  Bracket   Cross        Correct Tag
+ID  Len.  Stat. Recal  Prec.  Bracket gold test Bracket Words  Tags Accracy
+============================================================================
+1   37    0   77.27  65.38    17     22   26      5     34    27    79.41
+2   21    0   69.23  64.29     9     13   14      2     20    16    80.00
+3   47    0   80.00  82.35    28     35   34      4     44    40    90.91
+4   26    0   35.29  37.50     6     17   16      8     25    18    72.00
+5   44    0   42.31  33.33    11     26   33     17     38    28    73.68
+============================================================================
+              62.83  57.72    71   113   123      0    161   129    80.12
+=== Summary ===
+
+-- All --
+Number of sentence        =      5
+Number of Error sentence  =      0
+Number of Skip  sentence  =      0
+Number of Valid sentence  =      5
+Bracketing Recall         =  62.83
+Bracketing Precision      =  57.72
+Bracketing FMeasure       =  60.17
+Complete match            =   0.00
+Average crossing          =   7.20
+No crossing               =   0.00
+2 or less crossing        =  20.00
+Tagging accuracy          =  80.12
+
+-- len<=40 --
+Number of sentence        =      3
+Number of Error sentence  =      0
+Number of Skip  sentence  =      0
+Number of Valid sentence  =      3
+Bracketing Recall         =  61.54
+Bracketing Precision      =  57.14
+Bracketing FMeasure       =  59.26
+Complete match            =   0.00
+Average crossing          =   5.00
+No crossing               =   0.00
+2 or less crossing        =  33.33
+Tagging accuracy          =  77.22

parsing/EVALB/bug/bug.rsl-old

@@ -0,0 +1,45 @@
+Sent.                        Matched  Bracket   Cross        Correct Tag
+ID  Len.  Stat. Recal  Prec.  Bracket gold test Bracket Words  Tags Accracy
+============================================================================
+1 : Length unmatch (33|35)
+   1   37    1    0.00   0.00     0      0    0      0      0     0     0.00
+2 : Length unmatch (19|21)
+   2   21    1    0.00   0.00     0      0    0      0      0     0     0.00
+3 : Length unmatch (44|45)
+   3   47    1    0.00   0.00     0      0    0      0      0     0     0.00
+4 : Length unmatch (24|26)
+   4   26    1    0.00   0.00     0      0    0      0      0     0     0.00
+5 : Length unmatch (38|39)
+   5   44    1    0.00   0.00     0      0    0      0      0     0     0.00
+============================================================================
+      0     0     0.00
+
+=== Summary ===
+
+-- All --
+Number of sentence        =      5
+Number of Error sentence  =      5
+Number of Skip  sentence  =      0
+Number of Valid sentence  =      0
+Bracketing Recall         =   0.00
+Bracketing Precision      =   0.00
+Bracketing FMeasure       =    nan
+Complete match            =   0.00
+Average crossing          =   0.00
+No crossing               =   0.00
+2 or less crossing        =   0.00
+Tagging accuracy          =   0.00
+
+-- len<=40 --
+Number of sentence        =      3
+Number of Error sentence  =      3
+Number of Skip  sentence  =      0
+Number of Valid sentence  =      0
+Bracketing Recall         =   0.00
+Bracketing Precision      =   0.00
+Bracketing FMeasure       =    nan
+Complete match            =   0.00
+Average crossing          =   0.00
+No crossing               =   0.00
+2 or less crossing        =   0.00
+Tagging accuracy          =   0.00

parsing/EVALB/bug/bug.tst

@@ -0,0 +1,5 @@
+(S1 (S (NP (DT The) (JJ Thy-1) (NN gene) (NN promoter)) (VP (VP (VBZ resembles) (NP (NP (DT a) (ADJP (CD ") (NN housekeeping)) (NN ") (NN promoter)) (SBAR (WHPP (IN in) (WHNP (WDT that))) (S (NP (PRP it)) (VP (VBZ is) (VP (VBN located) (PP (IN within) (NP (DT a) (JJ methylation-free) (NN island))))))))) (, ,) (VP (VBZ lacks) (NP (DT a) (JJ canonical) (NNP TATA) (NN box))) (, ,) (CC and) (VP (VBZ displays) (NP (NP (NN heterogeneity)) (PP (IN in) (NP (NP (DT the) (JJ 5'-end) (NNS termini)) (PP (IN of) (NP (DT the) (NN mRNA)))))))) (. .)))
+(S1 (S (NP (NP (DT The) (JJ latter) (CD ") (JJ nuclear) (NN factor)) (PP (IN for) (NP (VBN activated) (NN T) (NNS cells)))) (VP (VBZ ") (ADJP (JJ likely) (S (VP (VBZ contributes) (PP (TO to) (NP (NP (DT the) (NN tissue) (NN specificity)) (PP (IN of) (NP (JJ IL-2) (NN gene) (NN expression))))))))) (. .)))
+(S1 (S (ADVP (RB Thus)) (, ,) (NP (PRP we)) (VP (VBD postulated) (SBAR (SBAR (IN that) (S (NP (NP (DT the) (JJ circadian) (NN modification)) (PP (IN of) (NP (NNP GR)))) (VP (VBD was) (ADJP (JJ independent) (PP (IN of) (NP (DT the) (JJ diurnal) (NNS fluctuations)))) (PP (IN in) (NP (NP (NN plasma) (JJ cortisol) (NN level)) (CC or) (NP (NP (DT the) (JJ circadian) (NNS variations)) (PP (IN in) (NP (JJ environmental) (NN lighting))))))))) (CC and) (SBAR (IN that) (S (NP (DT the) (NN rhythmicity)) (VP (MD might) (VP (VB be) (VP (VBN regulated) (PP (IN by) (NP (DT the) ('' ') (NP (JJ circadian) (NN pacemaker) (POS ')) (VP (VBN located) (PP (IN in) (NP (DT the) (JJ human) (JJ basal) (NN brain))))))))))))) (. .)))
+(S1 (S (NP (JJ Such) (NN transcription) (NNS factors)) (VP (VBP play) (NP (NP (DT a) (JJ key) (NN role)) (PP (IN in) (NP (NP (DT the) (NN development)) (PP (IN of) (NP (NP (DT the) (JJ mature) (JJ T-cell) (NN phenotype)) (PP (IN by) (NP (NP (NN functioning) (RB as) (POS ')) (NN master) (NNS regulators))))) (PP (IN of) (NP (JJ T-cell) (NN differentiation) (POS '))))))) (. .)))
+(S1 (S (NP (NP (DT The) (NN conversion)) (PP (IN of) (NP (DT the)))) (VP (VBD TCEd) (PP (TO to) (NP (NP (DT a) ('' ') (JJ perfect) ('' ') (NN NF-kB)) (SBAR (S (NP (JJ binding) (NN site)) (VP (VBZ leads) (PP (TO to) (NP (NP (NP (DT a) (ADJP (RBR tighter) (JJ binding)) (PP (IN of) (NP (NP (NNS NF-kB)) (PP (PP (TO to) (NP (JJ TCEd) (NN DNA))) (CC and) (PP (, ,) (PP (IN as) (NP (DT a) (JJ functional) (NN consequence))) (, ,) (TO to) (NP (NP (DT the) (NN activity)) (PP (IN of) (NP (DT the)))))))) (POS ')) (JJ converted) ('' ') (JJ TCEd) (NNS motifs)) (PP (IN in) (NP (NNP HeLa) (NNS cells))))))))))) (. .)))

parsing/EVALB/evalb.c

@@ -0,0 +1,1537 @@
+/*****************************************************************/
+/* evalb [-p param_file] [-dh] [-e n] gold-file test-file        */
+/*                                                               */
+/*        Evaluate bracketing in test-file against gold-file.    */
+/*        Return recall, precision, tagging accuracy.            */
+/*                                                               */
+/*   <option>                                                    */
+/*        -p param_file  parameter file                          */
+/*        -d             debug mode                              */
+/*        -e n           number of error to kill (default=10)    */
+/*        -h             help                                    */
+/*                                                               */
+/*                                         Satoshi Sekine (NYU)  */
+/*                                         Mike Collins (UPenn)  */
+/*                                                               */
+/*                                         October.1997          */
+/*                                                               */
+/* Please refer README for the update information                */
+/*****************************************************************/
+
+#include <stdio.h>
+#include <stdlib.h> //### added for exit, atoi decls
+#include <ctype.h>
+#include <string.h>
+
+
+/* Internal Data format -------------------------------------------*/
+/*                                                                 */
+/* (S (NP (NNX this)) (VP (VBX is) (NP (DT a) (NNX pen))) (SYM .)) */
+/*                                                                 */
+/*   wn=5                                                          */
+/*                        word    label                            */
+/*   terminal[0] =        this     NNX                             */
+/*   terminal[1] =        is       VBX                             */
+/*   terminal[2] =        a        DT                              */
+/*   terminal[3] =        pen      NNX                             */
+/*   terminal[4] =        .        SYM                             */
+/*                                                                 */
+/*   bn=4                                                          */
+/*                      start     end      label                   */
+/*   bracket[0]  =        0        5         S                     */
+/*   bracket[1]  =        0        0         NP                    */
+/*   bracket[2]  =        1        4         VP                    */
+/*   bracket[3]  =        2        4         NP                    */
+/*                                                                 */
+/*              matched bracketing                                 */
+/*   Recall = ---------------------------                          */
+/*             # of bracket in ref-data                            */
+/*                                                                 */
+/*              matched bracketing                                 */
+/*   Recall = ---------------------------                          */
+/*             # of bracket in test-data                           */
+/*                                                                 */
+/*-----------------------------------------------------------------*/
+
+/******************/
+/* constant macro */
+/******************/
+
+#define MAX_SENT_LEN           5000
+#define MAX_WORD_IN_SENT        200
+#define MAX_BRACKET_IN_SENT     200
+#define MAX_WORD_LEN            100
+#define MAX_LABEL_LEN            30
+#define MAX_QUOTE_TERM           20
+
+#define MAX_DELETE_LABEL        100
+#define MAX_EQ_LABEL            100
+#define MAX_EQ_WORD             100
+
+#define MAX_LINE_LEN            500
+
+#define DEFAULT_MAX_ERROR        10
+#define DEFAULT_CUT_LEN          40
+
+/*************/
+/* structure */
+/*************/
+
+typedef struct ss_terminal {
+    char word[MAX_WORD_LEN];
+    char label[MAX_LABEL_LEN];
+    int  result;                /* 0:unmatch, 1:match, 9:undef */
+} s_terminal;
+
+typedef struct ss_term_ind {
+	s_terminal term;
+	int index;
+   int bracket;
+   int endslen;
+   int ends[MAX_BRACKET_IN_SENT];
+} s_term_ind;
+
+typedef struct ss_bracket {
+    int start;
+    int end;
+    unsigned int buf_start;
+    unsigned int buf_end;
+    char label[MAX_LABEL_LEN];
+    int  result;                 /* 0: unmatch, 1:match, 5:delete 9:undef */
+} s_bracket;
+
+
+typedef struct ss_equiv {
+    char *s1;
+    char *s2;
+} s_equiv;
+
+
+/****************************/
+/* global variables         */
+/*   gold-data: suffix = 1  */
+/*   test-data: suffix = 2  */
+/****************************/
+
+/*---------------*/
+/* Sentence data */
+/*---------------*/
+int wn1, wn2;                              /* number of words in sentence  */
+int r_wn1;                                 /* number of words in sentence  */
+                                           /* which only ignores labels in */
+                                           /* DELETE_LABEL_FOR_LENGTH      */
+
+s_terminal terminal1[MAX_WORD_IN_SENT];    /* terminal information */
+s_terminal terminal2[MAX_WORD_IN_SENT];
+
+s_term_ind quotterm1[MAX_QUOTE_TERM];      /* special terminals ("'","POS") */
+s_term_ind quotterm2[MAX_QUOTE_TERM];
+
+int bn1, bn2;                              /* number of brackets */
+
+int r_bn1, r_bn2;                          /* number of brackets */
+                                           /* after deletion */
+
+s_bracket bracket1[MAX_BRACKET_IN_SENT];   /* bracket information */
+s_bracket bracket2[MAX_BRACKET_IN_SENT];
+
+
+/*------------*/
+/* Total data */
+/*------------*/
+int TOTAL_bn1, TOTAL_bn2, TOTAL_match;     /* total number of brackets */
+int TOTAL_sent;                            /* No. of sentence */
+int TOTAL_error_sent;                      /* No. of error sentence */
+int TOTAL_skip_sent;                       /* No. of skip sentence */
+int TOTAL_comp_sent;                       /* No. of complete match sent */
+int TOTAL_word;                            /* total number of word */
+int TOTAL_crossing;                        /* total crossing */
+int TOTAL_no_crossing;                     /* no crossing sentence */
+int TOTAL_2L_crossing;                     /* 2 or less crossing sentence */
+int TOTAL_correct_tag;                     /* total correct tagging */
+
+int TOT_cut_len = DEFAULT_CUT_LEN;         /* Cut-off length in statistics */
+
+                                 /* data for sentences with len <= CUT_LEN */
+                                 /* Historically it was 40.                */
+int TOT40_bn1, TOT40_bn2, TOT40_match;     /* total number of brackets */
+int TOT40_sent;                            /* No. of sentence */
+int TOT40_error_sent;                      /* No. of error sentence */
+int TOT40_skip_sent;                       /* No. of skip sentence */
+int TOT40_comp_sent;                       /* No. of complete match sent */
+int TOT40_word;                            /* total number of word */
+int TOT40_crossing;                        /* total crossing */
+int TOT40_no_crossing;                     /* no crossing sentence */
+int TOT40_2L_crossing;                     /* 2 or less crossing sentence */
+int TOT40_correct_tag;                     /* total correct tagging */
+
+/*------------*/
+/* miscallous */
+/*------------*/
+int Line;                                  /* line number */
+int Error_count = 0;                       /* Error count */
+int Status;                                /* Result status for each sent */
+                                           /*    0: OK, 1: skip, 2: error */
+
+/*-------------------*/
+/* stack manuplation */
+/*-------------------*/
+int stack_top;
+int stack[MAX_BRACKET_IN_SENT];
+
+/************************************************************/
+/* User parameters which can be specified in parameter file */
+/************************************************************/
+
+/*------------------------------------------*/
+/* Debug mode                               */
+/*   print out data for individual sentence */
+/*------------------------------------------*/
+int DEBUG=0;
+
+/*------------------------------------------*/
+/* MAX error                                */
+/*    Number of error to stop the process.  */
+/*    This is useful if there could be      */
+/*    tokanization error.                   */
+/*    The process will stop when this number*/
+/*    of errors are accumulated.            */
+/*------------------------------------------*/
+int Max_error = DEFAULT_MAX_ERROR;
+
+/*------------------------------------------*/
+/* Cut-off length for statistics            */
+/*    int TOT_cut_len = DEFAULT_CUT_LEN;    */
+/*    (Defined above)                       */
+/*------------------------------------------*/
+
+
+/*------------------------------------------*/
+/* unlabeled or labeled bracketing          */
+/*    0: unlabeled bracketing               */
+/*    1: labeled bracketing                 */
+/*------------------------------------------*/
+int F_label    = 1;                 
+
+/*------------------------------------------*/
+/* Delete labels                            */
+/*    list of labels to be ignored.         */
+/*    If it is a pre-terminal label, delete */
+/*    the word along with the brackets.     */
+/*    If it is a non-terminal label, just   */
+/*    delete the brackets (don't delete     */
+/*    childrens).                           */
+/*------------------------------------------*/
+char *Delete_label[MAX_DELETE_LABEL];
+int Delete_label_n = 0;
+
+/*------------------------------------------*/
+/* Delete labels for length calculation     */
+/*    list of labels to be ignored for      */
+/*    length calculation purpose            */
+/*------------------------------------------*/
+char *Delete_label_for_length[MAX_DELETE_LABEL];
+int Delete_label_for_length_n = 0;
+
+/*------------------------------------------*/
+/* Labels to be considered for misquote     */
+/*    (could be possesive or quote)         */
+/*------------------------------------------*/
+char *Quote_term[MAX_QUOTE_TERM];
+int Quote_term_n = 0;
+
+/*------------------------------------------*/
+/* Equivalent labels, words                 */
+/*     the pairs are considered equivalent  */
+/*     This is non-directional.             */
+/*------------------------------------------*/
+s_equiv EQ_label[MAX_EQ_LABEL];
+int EQ_label_n = 0;
+
+s_equiv EQ_word[MAX_EQ_WORD];
+int EQ_word_n = 0;
+
+
+
+/************************/
+/* Function return-type */
+/************************/
+int main();
+void init_global();
+void print_head();
+void init();
+void read_parameter_file();
+void set_param();
+int narg();
+int read_line();
+
+void pushb();
+int popb();
+int stackempty();
+
+void calc_result(unsigned char *buf1,unsigned char *buf);
+void fix_quote();
+void reinsert_term();
+void massage_data();
+void modify_label();
+void individual_result();
+void print_total();
+void dsp_info();
+int is_terminator();
+int is_deletelabel();
+int is_deletelabel_for_length();
+int is_quote_term();
+int word_comp();
+int label_comp();
+
+void Error();
+void Fatal();
+void Usage();
+
+/* ### provided by std headers 
+int fprintf();
+int printf();
+int atoi();
+int fclose();
+int sscanf();
+*/
+
+/***********/
+/* program */
+/***********/
+#define ARG_CHECK(st) if(!(*++(*argv) || (--argc && *++argv))){ \
+			 fprintf(stderr,"Missing argument: %s\n",st); \
+		      }
+
+int
+main(argc,argv)
+int argc;
+char *argv[];
+{
+    char *filename1, *filename2;
+    FILE *fd1, *fd2;
+    unsigned char buff[5000];
+    unsigned char buff1[5000];
+
+    filename1=NULL;
+    filename2=NULL;
+
+    for(argc--,argv++;argc>0;argc--,argv++){
+	if(**argv == '-'){
+	    while(*++(*argv)){
+		switch(**argv){
+
+		  case 'h':    /* help */
+		    Usage();
+		    exit(1);
+
+		  case 'd':      /* debug mode */
+		    DEBUG = 1;
+		    goto nextarg;
+
+		  case 'D':      /* debug mode */
+		    DEBUG = 2;
+		    goto nextarg;
+
+		  case 'c':      /* cut-off length */
+		    ARG_CHECK("cut-off length for statistices");
+		    TOT_cut_len = atoi(*argv);
+		    goto nextarg;
+
+		  case 'e':      /* max error */
+		    ARG_CHECK("number of error to kill");
+		    Max_error = atoi(*argv);
+		    goto nextarg;
+
+		  case 'p':      /* parameter file */
+		    ARG_CHECK("parameter file");
+		    read_parameter_file(*argv);
+		    goto nextarg;
+
+		  default:
+		    Usage();
+		    exit(0);
+		}
+	    }
+	} else {
+	    if(filename1==NULL){
+		filename1 = *argv;
+	    }else if(filename2==NULL){
+		filename2 = *argv;
+	    }
+	}
+      nextarg: continue;
+    }
+
+    init_global();
+
+
+    if((fd1 = fopen(filename1,"r"))==NULL){
+	Fatal("Can't open gold file (%s)\n",filename1);
+    }
+    if((fd2 = fopen(filename2,"r"))==NULL){
+	Fatal("Can't open test file (%s)\n",filename2);
+    }
+
+    print_head();
+
+    for(Line=1;fgets(buff,5000,fd1)!=NULL;Line++){
+    
+	init();
+
+      /* READ 1 */
+	r_wn1 = read_line(buff,terminal1,quotterm1,&wn1,bracket1,&bn1);
+
+	strcpy(buff1,buff);
+
+      /* READ 2 */
+	if(fgets(buff,5000,fd2)==NULL){
+	    Error("Number of lines unmatch (too many lines in gold file)\n");
+	    break;
+	}
+
+	read_line(buff,terminal2,quotterm2,&wn2,bracket2,&bn2);
+
+      /* Calculate result and print it */
+	calc_result(buff1,buff);
+
+	if(DEBUG>=1){
+	    dsp_info();
+	}
+    }
+
+    if(fgets(buff,5000,fd2)!=NULL){
+	Error("Number of lines unmatch (too many lines in test file)\n");
+    }
+
+    print_total();
+
+    return (0);
+}
+
+
+/*-----------------------------*/
+/* initialize global variables */
+/*-----------------------------*/
+void
+init_global()
+{
+    TOTAL_bn1 = TOTAL_bn2 = TOTAL_match = 0;
+    TOTAL_sent = TOTAL_error_sent = TOTAL_skip_sent = TOTAL_comp_sent = 0;
+    TOTAL_word = TOTAL_correct_tag = 0;
+    TOTAL_crossing = 0;
+    TOTAL_no_crossing = TOTAL_2L_crossing = 0;
+
+    TOT40_bn1 = TOT40_bn2 = TOT40_match = 0;
+    TOT40_sent = TOT40_error_sent = TOT40_skip_sent = TOT40_comp_sent = 0;
+    TOT40_word = TOT40_correct_tag = 0;
+    TOT40_crossing = 0;
+    TOT40_no_crossing = TOT40_2L_crossing = 0;
+
+}
+
+
+/*------------------*/
+/* print head title */
+/*------------------*/
+void
+print_head()
+{
+    printf("  Sent.                        Matched  Bracket   Cross        Correct Tag\n");
+    printf(" ID  Len.  Stat. Recal  Prec.  Bracket gold test Bracket Words  Tags Accracy\n");
+    printf("============================================================================\n");
+}
+
+
+/*-----------------------------------------------*/
+/* initialization at each individual computation */
+/*-----------------------------------------------*/
+void
+init()
+{
+  int i;
+
+  wn1 = 0;
+  wn2 = 0;
+  bn1 = 0;
+  bn2 = 0;
+  r_bn1 = 0;
+  r_bn2 = 0;
+
+  for(i=0;i<MAX_WORD_IN_SENT;i++){
+      terminal1[i].word[0]  = '\0';
+      terminal1[i].label[0] = '\0';
+      terminal1[i].result   = 9;
+      terminal2[i].word[0]  = '\0';
+      terminal2[i].label[0] = '\0';
+      terminal2[i].result   = 9;
+  }
+
+  for(i=0;i<MAX_QUOTE_TERM;i++){
+      quotterm1[i].term.word[0]  = '\0';
+      quotterm1[i].term.label[0] = '\0';
+      quotterm1[i].term.result   = 9;
+      quotterm1[i].index         = -1;
+      quotterm1[i].bracket       = -1;
+      quotterm2[i].term.word[0]  = '\0';
+      quotterm2[i].term.label[0] = '\0';
+      quotterm2[i].term.result   = 9;
+      quotterm2[i].index         = -1;
+      quotterm2[i].bracket       = -1;
+  }
+
+  for(i=0;i<MAX_BRACKET_IN_SENT;i++){
+      bracket1[i].start    = -1;
+      bracket1[i].end      = -1;
+      bracket1[i].label[0] = '\0';
+      bracket1[i].result   = 9;
+      bracket2[i].start    = -1;
+      bracket2[i].end      = -1;
+      bracket2[i].label[0] = '\0';
+      bracket2[i].result   = 9;
+  }
+
+  Status = 0;
+}
+
+/*----------------*/
+/* parameter file */
+/*----------------*/
+void
+read_parameter_file(filename)
+char *filename;
+{
+    char buff[MAX_LINE_LEN];
+    FILE *fd;
+    int line;
+    int i;
+
+    if((fd=fopen(filename,"r"))==NULL){
+	Fatal("Can't open parameter file (%s)\n",filename);
+    }
+
+    for(line=1;fgets(buff,MAX_LINE_LEN,fd)!=NULL;line++){
+
+      /* clean up the tail and find unvalid line */
+      /*-----------------------------------------*/
+	for(i=strlen(buff)-1;i>0 && (isspace(buff[i]) || buff[i]=='\n');i--){
+	    buff[i]='\0';
+	}
+	if(buff[0]=='#' ||      /* comment-line */
+	   strlen(buff)<3){     /* too short, just ignore */
+	    continue;
+	}
+
+      /* place the parameter and value */
+      /*-------------------------------*/
+	for(i=0;!isspace(buff[i]);i++);
+	for(;isspace(buff[i]) && buff[i]!='\0';i++);
+	if(buff[i]=='\0'){
+	    fprintf(stderr,"Empty value in parameter file (%d)\n",line);
+	}
+
+      /* set parameter and value */
+      /*-------------------------*/
+	set_param(buff,buff+i);
+    }
+
+    fclose(fd);
+}
+
+
+#define STRNCMP(s) (strncmp(param,s,strlen(s))==0 &&  \
+		    (param[strlen(s)]=='\0' || isspace(param[strlen(s)])))
+
+
+void
+set_param(param,value)
+char *param, *value;
+{
+    char l1[MAX_LABEL_LEN], l2[MAX_LABEL_LEN];
+
+    if(STRNCMP("DEBUG")){
+
+	DEBUG = atoi(value);
+
+    }else if(STRNCMP("MAX_ERROR")){
+
+	Max_error = atoi(value);
+
+    }else if(STRNCMP("CUTOFF_LEN")){
+
+	TOT_cut_len = atoi(value);
+
+    }else if(STRNCMP("LABELED")){
+
+	F_label = atoi(value);
+
+    }else if(STRNCMP("DELETE_LABEL")){
+
+	Delete_label[Delete_label_n] = (char *)malloc(strlen(value)+1);
+	strcpy(Delete_label[Delete_label_n],value);
+	Delete_label_n++;
+
+    }else if(STRNCMP("DELETE_LABEL_FOR_LENGTH")){
+
+	Delete_label_for_length[Delete_label_for_length_n] = (char *)malloc(strlen(value)+1);
+	strcpy(Delete_label_for_length[Delete_label_for_length_n],value);
+	Delete_label_for_length_n++;
+
+    }else if(STRNCMP("QUOTE_LABEL")){
+
+	Quote_term[Quote_term_n] = (char *)malloc(strlen(value)+1);
+	strcpy(Quote_term[Quote_term_n],value);
+	Quote_term_n++;
+
+    }else if(STRNCMP("EQ_LABEL")){
+
+	if(narg(value)!=2){
+	    fprintf(stderr,"EQ_LABEL requires two values\n");
+	    return;
+	}
+	sscanf(value,"%s %s",l1,l2);
+	EQ_label[EQ_label_n].s1 = (char *)malloc(strlen(l1)+1);
+	strcpy(EQ_label[EQ_label_n].s1,l1);
+	EQ_label[EQ_label_n].s2 = (char *)malloc(strlen(l2)+1);
+	strcpy(EQ_label[EQ_label_n].s2,l2);
+	EQ_label_n++;
+
+    }else if(STRNCMP("EQ_WORD")){
+
+	if(narg(value)!=2){
+	    fprintf(stderr,"EQ_WORD requires two values\n");
+	    return;
+	}
+	sscanf(value,"%s %s",l1,l2);
+	EQ_word[EQ_word_n].s1 = (char *)malloc(strlen(l1)+1);
+	strcpy(EQ_word[EQ_word_n].s1,l1);
+	EQ_word[EQ_word_n].s2 = (char *)malloc(strlen(l2)+1);
+	strcpy(EQ_word[EQ_word_n].s2,l2);
+	EQ_word_n++;
+
+    }else{
+
+	fprintf(stderr,"Unknown keyword (%s) in parameter file\n",param);
+
+    }
+}
+
+
+int
+narg(s)
+char *s;
+{
+    int n;
+
+    for(n=0;*s!='\0';){
+	for(;isspace(*s);s++);
+	if(*s=='\0'){
+	    break;
+	}
+	n++;
+	for(;!isspace(*s);s++){
+	    if(*s=='\0'){
+		break;
+	    }
+	}
+    }
+
+    return(n);
+}
+
+/*-----------------------------*/
+/* Read line and gather data.  */
+/* Return langth of sentence.  */
+/*-----------------------------*/
+int
+read_line(buff, terminal, quotterm, wn, bracket, bn)
+char *buff;
+s_terminal terminal[];
+s_term_ind quotterm[];
+int *wn;
+s_bracket bracket[];
+int *bn;
+{
+    char *p, *q, label[MAX_LABEL_LEN], word[MAX_WORD_LEN];
+    int	  qt;		  /* quote term counter */
+    int   wid, bid;       /* word ID, bracket ID */
+    int   n;              /* temporary remembering the position */
+    int   b;              /* temporary remembering bid */
+    int   i;
+    int   len;            /* length of the sentence */
+
+    len = 0;
+    stack_top=0;
+
+    for(p=buff,qt=0,wid=0,bid=0;*p!='\0';){
+
+	if(isspace(*p)){
+	    p++;
+	    continue;
+
+        /* open bracket */
+        /*--------------*/
+	}else if(*p=='('){
+
+	    n=wid;
+	    for(p++,i=0;!is_terminator(*p);p++,i++){
+		label[i]=*p;
+	    }
+	    label[i]='\0';
+
+	    /* Find terminals */
+	    q = p;
+	    if(isspace(*q)){
+		for(q++;isspace(*q);q++);
+		for(i=0;!is_terminator(*q);q++,i++){
+		    word[i]=*q;
+		}
+		word[i]='\0';
+
+                /* compute length */
+		if(*q==')' && !is_deletelabel_for_length(label)==1){
+		    len++;
+		}
+      if (DEBUG>1)
+         printf("label=%s, word=%s, wid=%d\n",label,word,wid);
+		/* quote terminal */
+		if(*q==')' && is_quote_term(label,word)==1){
+			strcpy(quotterm[qt].term.word,word);
+			strcpy(quotterm[qt].term.label,label);
+			quotterm[qt].index = wid;
+         quotterm[qt].bracket = bid;
+         quotterm[qt].endslen = stack_top;
+         //quotterm[qt].ends = (int*)malloc(stack_top*sizeof(int));
+         memcpy(quotterm[qt].ends,stack,stack_top*sizeof(int));
+			qt++;
+		}
+		
+                /* delete terminal */
+		if(*q==')' && is_deletelabel(label)==1){
+		    p = q+1;
+		    continue;
+
+		/* valid terminal */
+		}else if(*q==')'){
+		    strcpy(terminal[wid].word,word);
+		    strcpy(terminal[wid].label,label);
+		    wid++;
+		    p = q+1;
+		    continue;
+
+                /* error */
+		}else if(*q!='('){
+		    Error("More than two elements in a bracket\n");
+		}
+	    }
+
+            /* otherwise non-terminal label */
+	    bracket[bid].start = wid;
+	    bracket[bid].buf_start = p-buff;
+	    strcpy(bracket[bid].label,label);
+	    pushb(bid);
+	    bid++;
+
+	/* close bracket */
+        /*---------------*/
+	}else if(*p==')'){
+
+	    b = popb();
+	    bracket[b].end = wid;
+	    bracket[b].buf_end = p-buff;
+	    p++;
+
+        /* error */
+        /*-------*/
+	}else{
+
+	    Error("Reading sentence\n");
+	}
+    }
+
+    if(!stackempty()){
+	Error("Bracketing is unbalanced (too many open bracket)\n");
+    }
+
+    *wn = wid;
+    *bn = bid;
+
+    return(len);
+}
+
+
+/*----------------------*/
+/* stack operation      */
+/* for bracketing pairs */
+/*----------------------*/
+void
+pushb(item)
+int item;
+{
+    stack[stack_top++]=item;
+}
+
+int
+popb()
+{
+    int item;
+
+    item = stack[stack_top-1];
+
+    if(stack_top-- < 0){
+	Error("Bracketing unbalance (too many close bracket)\n");
+    }
+    return(item);
+}
+
+int
+stackempty()
+{
+    if(stack_top==0){
+	return(1);
+    }else{
+	return(0);
+    }
+}
+
+
+/*------------------*/
+/* calculate result */
+/*------------------*/
+void
+calc_result(unsigned char *buf1,unsigned char *buf)
+{
+    int i, j, l;
+    int match, crossing, correct_tag;
+
+    int last_i = -1;
+
+    char my_buf[1000];
+    int match_found = 0;
+    
+    char match_j[200];
+    for (j = 0; j < bn2; ++j) {
+      match_j[j] = 0;
+    }
+
+    /* ML */
+    if (DEBUG>1)
+    	printf("\n");
+
+
+    /* Find skip and error */
+    /*---------------------*/
+    if(wn2==0){
+	Status = 2;
+	individual_result(0,0,0,0,0,0);
+	return;
+    }
+
+   if(wn1 != wn2){
+      //if (DEBUG>1)
+    //Error("Length unmatch (%d|%d)\n",wn1,wn2);
+	fix_quote();
+	if(wn1 != wn2){
+		Error("Length unmatch (%d|%d)\n",wn1,wn2);
+		individual_result(0,0,0,0,0,0);
+		return;
+	}
+    }
+
+    for(i=0;i<wn1;i++){
+	if(word_comp(terminal1[i].word,terminal2[i].word)==0){
+	    Error("Words unmatch (%s|%s)\n",terminal1[i].word,
+                                            terminal2[i].word);
+	    individual_result(0,0,0,0,0,0);
+	    return;
+	}
+    }
+
+    /* massage the data */
+    /*------------------*/
+    massage_data();
+	   
+    /* matching brackets */
+    /*-------------------*/
+    match = 0;
+    for(i=0;i<bn1;i++){
+      for(j=0;j<bn2;j++){
+
+         if (DEBUG>1)
+   printf("1.res=%d, 2.res=%d, 1.start=%d, 2.start=%d, 1.end=%d, 2.end=%d\n",bracket1[i].result,bracket2[j].result,bracket1[i].start,bracket2[j].start,bracket1[i].end,bracket2[j].end);
+
+	// does bracket match?
+	if(bracket1[i].result != 5 && 
+	   bracket2[j].result == 0 &&
+	   bracket1[i].start == bracket2[j].start && bracket1[i].end == bracket2[j].end) {
+
+	  // (1) do we not care about the label or (2) does the label match?
+	  if (F_label==0 || label_comp(bracket1[i].label,bracket2[j].label)==1) {
+	    bracket1[i].result = bracket2[j].result = 1;
+	    match++;
+	    match_found = 1;
+	    break;
+	  } else {
+	    if (DEBUG>1) {
+	      printf("  LABEL[%d-%d]: ",bracket1[i].start,bracket1[i].end-1);
+	      l = bracket1[i].buf_end-bracket1[i].buf_start;
+	      strncpy(my_buf,buf1+bracket1[i].buf_start,l);
+	      my_buf[l] = '\0';
+	      printf("%s\n",my_buf);
+	    }
+	    match_found = 1;
+	    match_j[j] = 1;
+	  }
+	}
+      }
+
+      if (!match_found && bracket1[i].result != 5 && DEBUG>1) {
+	/* ### ML 09/28/03: gold bracket with no corresponding test bracket */
+	printf("  BRACKET[%d-%d]: ",bracket1[i].start,bracket1[i].end-1);
+	l = bracket1[i].buf_end-bracket1[i].buf_start;
+	strncpy(my_buf,buf1+bracket1[i].buf_start,l);
+	my_buf[l] = '\0';
+	printf("%s\n",my_buf);
+      }
+      match_found = 0;
+    }
+
+    for(j=0;j<bn2;j++){
+      if (bracket2[j].result==0 && !match_j[j] && DEBUG>1) {
+	/* test bracket with no corresponding gold bracket */
+	printf("  EXTRA[%d-%d]: ",bracket2[j].start,bracket2[j].end-1);
+	l = bracket2[j].buf_end-bracket2[j].buf_start;
+	strncpy(my_buf,buf+bracket2[j].buf_start,l);
+	my_buf[l] = '\0';
+	printf("%s\n",my_buf);
+      }
+    }
+
+    /* crossing */
+    /*----------*/
+    crossing = 0;
+
+    /* crossing is counted based on the brackets */
+    /* in test rather than gold file (by Mike)   */
+    for(j=0;j<bn2;j++){
+      for(i=0;i<bn1;i++){
+	if(bracket1[i].result != 5 &&
+	   bracket2[j].result != 5 &&
+	   ((bracket1[i].start < bracket2[j].start &&
+	     bracket1[i].end   > bracket2[j].start &&
+	     bracket1[i].end   < bracket2[j].end) ||
+	    (bracket1[i].start > bracket2[j].start &&
+	     bracket1[i].start < bracket2[j].end &&
+	     bracket1[i].end   > bracket2[j].end))){
+
+	  /* ### ML 09/01/03: get details on cross-brackettings */
+	  if (i != last_i) {
+	    if (DEBUG>1) {
+	    	printf("  CROSSING[%d-%d]: ",bracket1[i].start,bracket1[i].end-1);
+	    	l = bracket1[i].buf_end-bracket1[i].buf_start;
+	    	strncpy(my_buf,buf1+bracket1[i].buf_start,l);
+	    	my_buf[l] = '\0';
+	    	printf("%s\n",my_buf);
+
+	    	/* ML
+	    	printf("\n  CROSSING at bracket %d:\n",i-1);
+	    	printf("  GOLD (tokens %d-%d): ",bracket1[i].start,bracket1[i].end-1);
+	    	l = bracket1[i].buf_end-bracket1[i].buf_start;
+	    	strncpy(my_buf,buf1+bracket1[i].buf_start,l);
+	    	my_buf[l] = '\0';
+	    	printf("%s\n",my_buf);
+	    	*/
+	    }
+	    last_i = i;
+	  }
+
+	  /* ML
+	  printf("  TEST (tokens %d-%d): ",bracket2[j].start,bracket2[j].end-1);
+	  l = bracket2[j].buf_end-bracket2[j].buf_start;
+	  strncpy(my_buf,buf+bracket2[j].buf_start,l);
+	  my_buf[l] = '\0';
+	  printf("%s\n",my_buf);
+	  */
+
+	  crossing++;
+	  break;
+	}
+      }
+    }
+
+    /* Tagging accuracy */
+    /*------------------*/
+    correct_tag=0;
+    for(i=0;i<wn1;i++){
+       if(label_comp(terminal1[i].label,terminal2[i].label)==1){
+          terminal1[i].result = terminal2[i].result = 1;
+          correct_tag++;
+       } else {
+          terminal1[i].result = terminal2[i].result = 0;
+       }
+    }
+
+    individual_result(wn1,r_bn1,r_bn2,match,crossing,correct_tag);
+}
+
+void
+fix_quote()
+{
+   int i,j,k;
+   if (DEBUG>1) {
+      for(i=0;i<MAX_QUOTE_TERM;i++){
+         if (quotterm1[i].index!=-1)
+            printf("%d: %s - %s\n",quotterm1[i].index,
+                  quotterm1[i].term.label,
+                  quotterm1[i].term.word);
+         if (quotterm2[i].index!=-1)
+            printf("%d: %s - %s\n",quotterm2[i].index,
+                  quotterm2[i].term.label,
+                  quotterm2[i].term.word);
+      }
+   }
+   for(i=0;i<MAX_QUOTE_TERM;i++) {
+      int ind = quotterm2[i].index;
+      if (ind!=-1) {
+         for(j=0;j<MAX_QUOTE_TERM;j++){
+            if (quotterm1[j].index==ind &&
+                  strcmp(quotterm1[j].term.label,
+                     quotterm2[i].term.label)!=0) {
+               if (is_deletelabel(quotterm1[j].term.label) && !is_deletelabel(quotterm2[i].term.label)) {
+                  reinsert_term(&quotterm1[j],terminal1,bracket1,&wn1);
+                  for (k=j;k<MAX_QUOTE_TERM;k++)
+                     if (quotterm1[k].index!=-1)
+                        quotterm1[k].index++;
+               } else if (is_deletelabel(quotterm2[i].term.label) && !is_deletelabel(quotterm1[j].term.label)) {
+                  reinsert_term(&quotterm2[i],terminal2,bracket2,&wn2);
+                  for (k=i;k<MAX_QUOTE_TERM;k++)
+                     if (quotterm2[k].index!=-1)
+                        quotterm2[k].index++;
+               }
+            }
+         }
+      } else break;
+   }
+}
+
+void
+reinsert_term(quot,terminal,bracket,wn)
+s_term_ind* quot;
+s_terminal terminal[];
+s_bracket bracket[];
+int* wn;
+{
+   int ind = quot->index;
+   int bra = quot->bracket;
+   s_terminal* term = &quot->term;
+   int k;
+   memmove(&terminal[ind+1],
+         &terminal[ind],
+         sizeof(s_terminal)*(MAX_WORD_IN_SENT-ind-1));
+   strcpy(terminal[ind].label,term->label);
+   strcpy(terminal[ind].word,term->word);
+   (*wn)++;
+   if (DEBUG>1)
+      printf("bra=%d, ind=%d\n",bra,ind);
+   for(k=0;k<MAX_BRACKET_IN_SENT;k++) {
+      if (bracket[k].start==-1)
+         break;
+      if (DEBUG>1)
+         printf("bracket[%d]={%d,%d}\n",k,bracket[k].start,bracket[k].end);
+      if (k>=bra) {
+         bracket[k].start++;
+         bracket[k].end++;
+      }
+      //if (bracket[k].start<=ind && bracket[k].end>=ind)
+         //bracket[k].end++;
+   }
+   if (DEBUG>1)
+      printf("endslen=%d\n",quot->endslen);
+   for(k=0;k<quot->endslen;k++) {
+      //printf("ends[%d]=%d",k,quot->ends[k]);
+      bracket[quot->ends[k]].end++;
+   }
+   //free(quot->ends);
+}
+/*
+void
+adjust_end(ind,bra)
+int ind;
+int bra;
+{
+    for(k=0;k<MAX_BRACKET_IN_SENT;k++) {
+      if (bracket[k].start==-1)
+         break;
+      printf("bracket[%d]={%d,%d}\n",k,bracket[k].start,bracket[k].end);
+      if (k>=bra)
+         bracket[k].end++;
+   }
+}
+*/
+void
+massage_data()
+{
+    int i, j;
+
+    /* for GOLD */
+    /*----------*/ 
+    for(i=0;i<bn1;i++){
+
+	bracket1[i].result = 0;
+
+	/* Zero element */
+	if(bracket1[i].start == bracket1[i].end){
+	    bracket1[i].result = 5;
+	    continue;
+	}
+
+        /* Modify label */
+	modify_label(bracket1[i].label);
+
+	/* Delete label */
+	for(j=0;j<Delete_label_n;j++){
+	    if(label_comp(bracket1[i].label,Delete_label[j])==1){
+		bracket1[i].result = 5;
+	    }
+	}
+    }
+	   
+    /* for TEST */
+    /*----------*/ 
+    for(i=0;i<bn2;i++){
+
+	bracket2[i].result = 0;
+
+	/* Zero element */
+	if(bracket2[i].start == bracket2[i].end){
+	    bracket2[i].result = 5;
+	    continue;
+	}
+
+        /* Modify label */
+	modify_label(bracket2[i].label);
+
+	/* Delete label */
+	for(j=0;j<Delete_label_n;j++){
+	    if(label_comp(bracket2[i].label,Delete_label[j])==1){
+		bracket2[i].result = 5;
+	    }
+	}
+    }
+
+
+    /* count up real number of brackets (exclude deleted ones) */
+    /*---------------------------------------------------------*/
+    r_bn1 = r_bn2 = 0;
+
+    for(i=0;i<bn1;i++){
+	if(bracket1[i].result != 5){
+	    r_bn1++;
+	}
+    }
+
+    for(i=0;i<bn2;i++){
+	if(bracket2[i].result != 5){
+	    r_bn2++;
+	}
+    }
+}
+
+
+/*------------------------*/
+/* trim the tail of label */
+/*------------------------*/
+void
+modify_label(label)
+char *label;
+{
+    char *p;
+
+    for(p=label;*p!='\0';p++){
+	if(*p=='-' || *p=='='){
+	    *p='\0';
+	    break;
+	}
+    }
+}
+
+
+/*-----------------------------------------------*/
+/* add individual statistics to TOTAL statictics */
+/*-----------------------------------------------*/
+void
+individual_result(wn1,bn1,bn2,match,crossing,correct_tag)
+int wn1,bn1,bn2,match,crossing,correct_tag;
+{
+
+    /* Statistics for ALL */
+    /*--------------------*/
+    TOTAL_sent++;
+    if(Status==1){
+	TOTAL_error_sent++;
+    }else if(Status==2){
+	TOTAL_skip_sent++;
+    }else{
+	TOTAL_bn1 += bn1;
+	TOTAL_bn2 += bn2;
+	TOTAL_match += match;
+	if(bn1==bn2 && bn2==match){
+	    TOTAL_comp_sent++;
+	}
+	TOTAL_word += wn1;
+	TOTAL_crossing += crossing;
+	if(crossing==0){
+	    TOTAL_no_crossing++;
+	}
+	if(crossing <= 2){
+	    TOTAL_2L_crossing++;
+	}
+	TOTAL_correct_tag += correct_tag;
+    }
+
+
+    /* Statistics for sent length <= TOT_cut_len */
+    /*-------------------------------------------*/
+    if(r_wn1<=TOT_cut_len){
+	TOT40_sent++;
+	if(Status==1){
+	    TOT40_error_sent++;
+	}else if(Status==2){
+	    TOT40_skip_sent++;
+	}else{
+	    TOT40_bn1 += bn1;
+	    TOT40_bn2 += bn2;
+	    TOT40_match += match;
+	    if(bn1==bn2 && bn2==match){
+		TOT40_comp_sent++;
+	    }
+	    TOT40_word += wn1;
+	    TOT40_crossing += crossing;
+	    if(crossing==0){
+		TOT40_no_crossing++;
+	    }
+	    if(crossing <= 2){
+		TOT40_2L_crossing++;
+	    }
+	    TOT40_correct_tag += correct_tag;
+	}
+    }
+
+    /* Print individual result */
+    /*-------------------------*/
+    printf("%4d  %3d    %d  ",Line,r_wn1,Status);
+    printf("%6.2f %6.2f   %3d    %3d  %3d    %3d",
+	   (r_bn1==0?0.0:100.0*match/r_bn1), 
+	   (r_bn2==0?0.0:100.0*match/r_bn2),
+	   match, r_bn1, r_bn2, crossing);
+
+    printf("   %4d  %4d   %6.2f\n",wn1,correct_tag,
+	   (wn1==0?0.0:100.0*correct_tag/wn1));
+}
+
+
+/*------------------------*/
+/* print total statistics */
+/*------------------------*/
+void
+print_total()
+{
+    int sentn;
+    double r,p,f;
+
+    printf("============================================================================\n");
+
+    if(TOTAL_bn1>0 && TOTAL_bn2>0){
+	printf("                %6.2f %6.2f %6d %5d %5d  %5d",
+	       (TOTAL_bn1>0?100.0*TOTAL_match/TOTAL_bn1:0.0),
+	       (TOTAL_bn2>0?100.0*TOTAL_match/TOTAL_bn2:0.0),
+	       TOTAL_match, 
+	       TOTAL_bn1, 
+	       TOTAL_bn2,
+	       TOTAL_crossing);
+    }
+
+    printf("  %5d %5d   %6.2f",
+	   TOTAL_word,
+	   TOTAL_correct_tag,
+	   (TOTAL_word>0?100.0*TOTAL_correct_tag/TOTAL_word:0.0));
+
+    printf("\n");
+    printf("=== Summary ===\n");
+
+    sentn = TOTAL_sent - TOTAL_error_sent - TOTAL_skip_sent;
+
+    printf("\n-- All --\n");
+    printf("Number of sentence        = %6d\n",TOTAL_sent);
+    printf("Number of Error sentence  = %6d\n",TOTAL_error_sent);
+    printf("Number of Skip  sentence  = %6d\n",TOTAL_skip_sent);
+    printf("Number of Valid sentence  = %6d\n",sentn);
+    
+    r = TOTAL_bn1>0 ? 100.0*TOTAL_match/TOTAL_bn1 : 0.0;
+    printf("Bracketing Recall         = %6.2f\n",r);
+
+    p = TOTAL_bn2>0 ? 100.0*TOTAL_match/TOTAL_bn2 : 0.0;
+    printf("Bracketing Precision      = %6.2f\n",p);
+
+    f = 2*p*r/(p+r);
+    printf("Bracketing FMeasure       = %6.2f\n",f);
+			    
+    printf("Complete match            = %6.2f\n",
+	   (sentn>0?100.0*TOTAL_comp_sent/sentn:0.0));
+    printf("Average crossing          = %6.2f\n",
+	   (sentn>0?1.0*TOTAL_crossing/sentn:0.0));
+    printf("No crossing               = %6.2f\n",
+	   (sentn>0?100.0*TOTAL_no_crossing/sentn:0.0));
+    printf("2 or less crossing        = %6.2f\n",
+	   (sentn>0?100.0*TOTAL_2L_crossing/sentn:0.0));
+    printf("Tagging accuracy          = %6.2f\n",
+	   (TOTAL_word>0?100.0*TOTAL_correct_tag/TOTAL_word:0.0));
+
+    sentn = TOT40_sent - TOT40_error_sent - TOT40_skip_sent;
+
+    printf("\n-- len<=%d --\n",TOT_cut_len);
+    printf("Number of sentence        = %6d\n",TOT40_sent);
+    printf("Number of Error sentence  = %6d\n",TOT40_error_sent);
+    printf("Number of Skip  sentence  = %6d\n",TOT40_skip_sent);
+    printf("Number of Valid sentence  = %6d\n",sentn);
+
+
+    r = TOT40_bn1>0 ? 100.0*TOT40_match/TOT40_bn1 : 0.0;
+    printf("Bracketing Recall         = %6.2f\n",r);
+
+    p = TOT40_bn2>0 ? 100.0*TOT40_match/TOT40_bn2 : 0.0;
+    printf("Bracketing Precision      = %6.2f\n",p);
+
+    f = 2*p*r/(p+r);
+    printf("Bracketing FMeasure       = %6.2f\n",f);
+
+    printf("Complete match            = %6.2f\n",
+	   (sentn>0?100.0*TOT40_comp_sent/sentn:0.0));
+    printf("Average crossing          = %6.2f\n",
+	   (sentn>0?1.0*TOT40_crossing/sentn:0.0));
+    printf("No crossing               = %6.2f\n",
+	   (sentn>0?100.0*TOT40_no_crossing/sentn:0.0));
+    printf("2 or less crossing        = %6.2f\n",
+	   (sentn>0?100.0*TOT40_2L_crossing/sentn:0.0));
+    printf("Tagging accuracy          = %6.2f\n",
+	   (TOT40_word>0?100.0*TOT40_correct_tag/TOT40_word:0.0));
+
+}
+
+
+/*--------------------------------*/
+/* display individual information */
+/*--------------------------------*/
+void
+dsp_info()
+{
+  int i, n;
+
+  printf("-<1>---(wn1=%3d, bn1=%3d)-           ",wn1,bn1);
+  printf("-<2>---(wn2=%3d, bn2=%3d)-\n",wn2,bn2);
+
+  n = (wn1>wn2?wn1:wn2);
+
+  for(i=0;i<n;i++){
+      if(terminal1[i].word[0]!='\0'){
+	  printf("%3d : %d : %-6s  %-16s      ",i,terminal1[i].result,
+		 terminal1[i].label,terminal1[i].word);
+      }else{
+	  printf("                                        ");
+      }
+
+      if(terminal2[i].word[0]!='\0'){
+	  printf("%3d : %d : %-6s  %-16s\n",i,terminal2[i].result,
+		 terminal2[i].label,terminal2[i].word);
+      }else{
+	  printf("\n");
+      }
+  }
+  printf("\n");
+
+  n = (bn1>bn2?bn1:bn2);
+
+  for(i=0;i<n;i++){
+      if(bracket1[i].start != -1){
+	  printf("%3d : %d : %3d  %3d  %-6s      ",i,bracket1[i].result,
+	                            bracket1[i].start,bracket1[i].end,
+	                            bracket1[i].label);
+      } else {
+	  printf("                                ");
+      }
+
+      if(bracket2[i].start != -1){
+	  printf("%3d : %d : %3d  %3d  %-6s\n",i,bracket2[i].result,
+                                    bracket2[i].start,bracket2[i].end,
+	                            bracket2[i].label);
+      } else {
+	  printf("\n");
+      }
+  }
+  printf("\n");
+
+  printf("========\n");
+
+}
+
+
+/*-----------------*/
+/* some predicates */
+/*-----------------*/
+int
+is_terminator(c)
+char c;
+{
+    if(isspace(c) || c=='(' || c==')'){
+	return(1);
+    }else{
+	return(0);
+    }
+}
+
+int
+is_deletelabel(s)
+char *s;
+{
+    int i;
+
+    for(i=0;i<Delete_label_n;i++){
+	if(strcmp(s,Delete_label[i])==0){
+	    return(1);
+	}
+    }
+
+    return(0);
+}
+
+int
+is_deletelabel_for_length(s)
+char *s;
+{
+    int i;
+
+    for(i=0;i<Delete_label_for_length_n;i++){
+	if(strcmp(s,Delete_label_for_length[i])==0){
+	    return(1);
+	}
+    }
+
+    return(0);
+}
+
+int
+is_quote_term(s,w)
+char *s;
+char *w;
+{
+    int i;
+
+    for(i=0;i<Quote_term_n;i++){
+		if(strcmp(s,Quote_term[i])==0){
+			if (strcmp(w,"'")==0 || strcmp(w,"\"")==0 || strcmp(w,"/")==0)
+	    	return(1);
+		}
+    }
+
+    return(0);
+}
+
+
+/*---------------*/
+/* compare words */
+/*---------------*/
+int
+word_comp(s1,s2)
+char *s1,*s2;
+{
+    int i;
+
+    if(strcmp(s1,s2)==0){
+	return(1);
+    }
+
+    for(i=0;i<EQ_word_n;i++){
+	if((strcmp(s1,EQ_word[i].s1)==0 &&
+	    strcmp(s2,EQ_word[i].s2)==0) ||
+	   (strcmp(s1,EQ_word[i].s2)==0 &&
+	    strcmp(s2,EQ_word[i].s1)==0)){
+	    return(1);
+	}
+    }
+
+    return(0);
+}
+
+/*----------------*/
+/* compare labels */
+/*----------------*/
+int
+label_comp(s1,s2)
+char *s1,*s2;
+{
+    int i;
+
+    if(strcmp(s1,s2)==0){
+	return(1);
+    }
+
+    for(i=0;i<EQ_label_n;i++){
+	if((strcmp(s1,EQ_label[i].s1)==0 &&
+	    strcmp(s2,EQ_label[i].s2)==0) ||
+	   (strcmp(s1,EQ_label[i].s2)==0 &&
+	    strcmp(s2,EQ_label[i].s1)==0)){
+	    return(1);
+	}
+    }
+
+    return(0);
+}
+
+
+/*--------*/
+/* errors */
+/*--------*/
+void
+Error(s,arg1,arg2,arg3)
+char *s, *arg1, *arg2, *arg3;
+{
+    Status = 1;
+    fprintf(stderr,"%d : ",Line);
+    fprintf(stderr,s,arg1,arg2,arg3);
+    if(Error_count++>Max_error){
+	exit(1);
+    }
+}
+
+
+/*---------------------*/
+/* fatal error to exit */
+/*---------------------*/
+void
+Fatal(s,arg1,arg2,arg3)
+char *s, *arg1, *arg2, *arg3;
+{
+    fprintf(stderr,s,arg1,arg2,arg3);
+    exit(1);
+}
+
+
+/*-------*/
+/* Usage */
+/*-------*/
+void
+Usage()
+{
+  fprintf(stderr," evalb [-dDh][-c n][-e n][-p param_file] gold-file test-file  \n");
+  fprintf(stderr,"                                                         \n");
+  fprintf(stderr,"    Evaluate bracketing in test-file against gold-file.  \n");
+  fprintf(stderr,"    Return recall, precision, F-Measure, tag accuracy.              \n");
+  fprintf(stderr,"                                                         \n");
+  fprintf(stderr,"  <option>                                               \n");
+  fprintf(stderr,"    -d             debug mode                            \n");
+  fprintf(stderr,"    -D             debug mode plus bracketing info       \n");
+  fprintf(stderr,"    -c n           cut-off length forstatistics (def.=40)\n");
+  fprintf(stderr,"    -e n           number of error to kill (default=10)  \n");
+  fprintf(stderr,"    -p param_file  parameter file                        \n");
+  fprintf(stderr,"    -h    help                                           \n");
+}

parsing/EVALB/new.prm

@@ -0,0 +1,87 @@
+##------------------------------------------##
+## Debug mode                               ##
+##   0: No debugging                        ##
+##   1: print data for individual sentence  ##
+##   2: print detailed bracketing info      ##
+##------------------------------------------##
+DEBUG 0
+
+##------------------------------------------##
+## MAX error                                ##
+##    Number of error to stop the process.  ##
+##    This is useful if there could be      ##
+##    tokanization error.                   ##
+##    The process will stop when this number##
+##    of errors are accumulated.            ##
+##------------------------------------------##
+MAX_ERROR 10
+
+##------------------------------------------##
+## Cut-off length for statistics            ##
+##    At the end of evaluation, the         ##
+##    statistics for the senetnces of length##
+##    less than or equal to this number will##
+##    be shown, on top of the statistics    ##
+##    for all the sentences                 ##
+##------------------------------------------##
+CUTOFF_LEN 40
+
+##------------------------------------------##
+## unlabeled or labeled bracketing          ##
+##    0: unlabeled bracketing               ##
+##    1: labeled bracketing                 ##
+##------------------------------------------##
+LABELED 1                 
+
+##------------------------------------------##
+## Delete labels                            ##
+##    list of labels to be ignored.         ##
+##    If it is a pre-terminal label, delete ##
+##    the word along with the brackets.     ##
+##    If it is a non-terminal label, just   ##
+##    delete the brackets (don't delete     ##
+##    deildrens).                           ##
+##------------------------------------------##
+DELETE_LABEL TOP
+DELETE_LABEL S1
+DELETE_LABEL -NONE-
+DELETE_LABEL ,
+DELETE_LABEL :
+DELETE_LABEL ``
+DELETE_LABEL ''
+DELETE_LABEL .
+DELETE_LABEL ?
+DELETE_LABEL !
+
+##------------------------------------------##
+## Delete labels for length calculation     ##
+##    list of labels to be ignored for      ##
+##    length calculation purpose            ##
+##------------------------------------------##
+DELETE_LABEL_FOR_LENGTH -NONE-
+
+##------------------------------------------##
+## Labels to be considered for misquote     ##
+##    (could be possesive or quote)         ##
+##------------------------------------------##
+QUOTE_LABEL ``
+QUOTE_LABEL ''
+QUOTE_LABEL POS
+
+##------------------------------------------##
+## These ones are less common, but          ##
+##    are on occasion output by parsers:    ##      
+##------------------------------------------##
+QUOTE_LABEL NN
+QUOTE_LABEL CD
+QUOTE_LABEL VBZ
+QUOTE_LABEL :
+
+##------------------------------------------##
+## Equivalent labels, words                 ##
+##     the pairs are considered equivalent  ##
+##     This is non-directional.             ##
+##------------------------------------------##
+EQ_LABEL ADVP PRT
+
+# EQ_WORD  Example example

parsing/EVALB/nk.prm

@@ -0,0 +1,92 @@
+# Based on new.prm (and by extension COLLINS.prm)
+# The only change from new.prm is increasing MAX_ERROR. The evaluation should be
+# identical to the standard setup, except that evalb won't give up early for a
+# parser that has just started training and does not yet produce good results.
+
+##------------------------------------------##
+## Debug mode                               ##
+##   0: No debugging                        ##
+##   1: print data for individual sentence  ##
+##   2: print detailed bracketing info      ##
+##------------------------------------------##
+DEBUG 0
+
+##------------------------------------------##
+## MAX error                                ##
+##    Number of error to stop the process.  ##
+##    This is useful if there could be      ##
+##    tokanization error.                   ##
+##    The process will stop when this number##
+##    of errors are accumulated.            ##
+##------------------------------------------##
+MAX_ERROR 10000
+
+##------------------------------------------##
+## Cut-off length for statistics            ##
+##    At the end of evaluation, the         ##
+##    statistics for the senetnces of length##
+##    less than or equal to this number will##
+##    be shown, on top of the statistics    ##
+##    for all the sentences                 ##
+##------------------------------------------##
+CUTOFF_LEN 40
+
+##------------------------------------------##
+## unlabeled or labeled bracketing          ##
+##    0: unlabeled bracketing               ##
+##    1: labeled bracketing                 ##
+##------------------------------------------##
+LABELED 1
+
+##------------------------------------------##
+## Delete labels                            ##
+##    list of labels to be ignored.         ##
+##    If it is a pre-terminal label, delete ##
+##    the word along with the brackets.     ##
+##    If it is a non-terminal label, just   ##
+##    delete the brackets (don't delete     ##
+##    deildrens).                           ##
+##------------------------------------------##
+DELETE_LABEL TOP
+DELETE_LABEL S1
+DELETE_LABEL -NONE-
+DELETE_LABEL ,
+DELETE_LABEL :
+DELETE_LABEL ``
+DELETE_LABEL ''
+DELETE_LABEL .
+DELETE_LABEL ?
+DELETE_LABEL !
+
+##------------------------------------------##
+## Delete labels for length calculation     ##
+##    list of labels to be ignored for      ##
+##    length calculation purpose            ##
+##------------------------------------------##
+DELETE_LABEL_FOR_LENGTH -NONE-
+
+##------------------------------------------##
+## Labels to be considered for misquote     ##
+##    (could be possesive or quote)         ##
+##------------------------------------------##
+QUOTE_LABEL ``
+QUOTE_LABEL ''
+QUOTE_LABEL POS
+
+##------------------------------------------##
+## These ones are less common, but          ##
+##    are on occasion output by parsers:    ##
+##------------------------------------------##
+QUOTE_LABEL NN
+QUOTE_LABEL CD
+QUOTE_LABEL VBZ
+QUOTE_LABEL :
+
+##------------------------------------------##
+## Equivalent labels, words                 ##
+##     the pairs are considered equivalent  ##
+##     This is non-directional.             ##
+##------------------------------------------##
+EQ_LABEL ADVP PRT
+
+# EQ_WORD  Example example

parsing/EVALB/sample/sample.gld

@@ -0,0 +1,24 @@
+(S (A (P this)) (B (Q is) (A (R a) (T test))))
+(S (A (P this)) (B (Q is) (A (R a) (T test))))
+(S (A (P this)) (B (Q is) (A (R a) (T test))))
+(S (A (P this)) (B (Q is) (A (R a) (T test))))
+(S (A (P this)) (B (Q is) (A (R a) (T test))))
+(S (A (P this)) (B (Q is) (A (R a) (T test))))
+(S (A (P this)) (B (Q is) (A (R a) (T test))))
+(S (A (P this)) (B (Q is) (A (R a) (T test))))
+(S (A (P this)) (B (Q is) (A (R a) (T test))))
+(S (A (P this)) (B (Q is) (A (R a) (T test))))
+(S (A (P this)) (B (Q is) (A (R a) (T test))))
+(S (A (P this)) (B (Q is) (A (R a) (T test))))
+(S (A (P this)) (B (Q is) (A (R a) (T test))))
+(S (A (P this)) (B (Q is) (A (R a) (T test))))
+(S (A (P this)) (B (Q is) (A (R a) (T test))))
+(S (A (P this)) (B (Q is) (A (R a) (T test))))
+(S (A (P this)) (B (Q is) (A (R a) (T test))))
+(S (A (P this)) (B (Q is) (A (R a) (T test))))
+(S (A (P this)) (B (Q is) (A (R a) (T test))))
+(S (A (P this)) (B (Q is) (A (R a) (T test))))
+(S (A-SBJ-1 (P this)) (B-WHATEVER (Q is) (A (R a) (T test))))
+(S (A (P this)) (B (Q is) (A (R a) (T test))) (A (P this)) (B (Q is) (A (R a) (T test))) (A (P this)) (B (Q is) (A (R a) (T test))) (A (P this)) (B (Q is) (A (R a) (T test))) (A (P this)) (B (Q is) (A (R a) (T test))) (A (P this)) (B (Q is) (A (R a) (T test))) (A (P this)) (B (Q is) (A (R a) (T test))) (A (P this)) (B (Q is) (A (R a) (T test))) (A (P this)) (B (Q is) (A (R a) (T test))) (A (P this)) (B (Q is) (A (R a) (T test))) (A (P this)) (B (Q is) (A (R a) (T test))))
+(S (A (P this)) (B (Q is) (A (R a) (T test))) (-NONE- *))
+(S (A (P this)) (B (Q is) (A (R a) (T test))) (: *))

parsing/EVALB/sample/sample.prm

@@ -0,0 +1,65 @@
+##------------------------------------------##
+## Debug mode                               ##
+##   print out data for individual sentence ##
+##------------------------------------------##
+DEBUG 0
+
+##------------------------------------------##
+## MAX error                                ##
+##    Number of error to stop the process.  ##
+##    This is useful if there could be      ##
+##    tokanization error.                   ##
+##    The process will stop when this number##
+##    of errors are accumulated.            ##
+##------------------------------------------##
+MAX_ERROR 10
+
+##------------------------------------------##
+## Cut-off length for statistics            ##
+##    At the end of evaluation, the         ##
+##    statistics for the senetnces of length##
+##    less than or equal to this number will##
+##    be shown, on top of the statistics    ##
+##    for all the sentences                 ##
+##------------------------------------------##
+CUTOFF_LEN 40
+
+##------------------------------------------##
+## unlabeled or labeled bracketing          ##
+##    0: unlabeled bracketing               ##
+##    1: labeled bracketing                 ##
+##------------------------------------------##
+LABELED 1                 
+
+##------------------------------------------##
+## Delete labels                            ##
+##    list of labels to be ignored.         ##
+##    If it is a pre-terminal label, delete ##
+##    the word along with the brackets.     ##
+##    If it is a non-terminal label, just   ##
+##    delete the brackets (don't delete     ##
+##    deildrens).                           ##
+##------------------------------------------##
+DELETE_LABEL TOP
+DELETE_LABEL -NONE-
+DELETE_LABEL ,
+DELETE_LABEL :
+DELETE_LABEL ``
+DELETE_LABEL ''
+
+##------------------------------------------##
+## Delete labels for length calculation     ##
+##    list of labels to be ignored for      ##
+##    length calculation purpose            ##
+##------------------------------------------##
+DELETE_LABEL_FOR_LENGTH -NONE-
+
+
+##------------------------------------------##
+## Equivalent labels, words                 ##
+##     the pairs are considered equivalent  ##
+##     This is non-directional.             ##
+##------------------------------------------##
+EQ_LABEL T TT
+
+EQ_WORD  This this

parsing/EVALB/sample/sample.rsl

@@ -0,0 +1,56 @@
+  Sent.                        Matched  Bracket   Cross        Correct Tag
+ ID  Len.  Stat. Recal  Prec.  Bracket gold test Bracket Words  Tags Accracy
+============================================================================
+   1    4    0  100.00 100.00     4      4    4      0      4     4   100.00
+   2    4    0   75.00  75.00     3      4    4      0      4     4   100.00
+   3    4    0  100.00 100.00     4      4    4      0      4     3    75.00
+   4    4    0   75.00  75.00     3      4    4      0      4     3    75.00
+   5    4    0   75.00  75.00     3      4    4      0      4     4   100.00
+   6    4    0   50.00  66.67     2      4    3      1      4     4   100.00
+   7    4    0   25.00 100.00     1      4    1      0      4     4   100.00
+   8    4    0    0.00   0.00     0      4    0      0      4     4   100.00
+   9    4    0  100.00  80.00     4      4    5      0      4     4   100.00
+  10    4    0  100.00  50.00     4      4    8      0      4     4   100.00
+  11    4    2    0.00   0.00     0      0    0      0      4     0     0.00
+  12    4    1    0.00   0.00     0      0    0      0      4     0     0.00
+  13    4    1    0.00   0.00     0      0    0      0      4     0     0.00
+  14    4    2    0.00   0.00     0      0    0      0      4     0     0.00
+  15    4    0  100.00 100.00     4      4    4      0      4     4   100.00
+  16    4    1    0.00   0.00     0      0    0      0      4     0     0.00
+  17    4    1    0.00   0.00     0      0    0      0      4     0     0.00
+  18    4    0  100.00 100.00     4      4    4      0      4     4   100.00
+  19    4    0  100.00 100.00     4      4    4      0      4     4   100.00
+  20    4    1    0.00   0.00     0      0    0      0      4     0     0.00
+  21    4    0  100.00 100.00     4      4    4      0      4     4   100.00
+  22   44    0  100.00 100.00    34     34   34      0     44    44   100.00
+  23    4    0  100.00 100.00     4      4    4      0      4     4   100.00
+  24    5    0  100.00 100.00     4      4    4      0      4     4   100.00
+============================================================================
+                 87.76  90.53     86     98   95     16    108   106    98.15
+=== Summary ===
+
+-- All --
+Number of sentence        =     24
+Number of Error sentence  =      5
+Number of Skip  sentence  =      2
+Number of Valid sentence  =     17
+Bracketing Recall         =  87.76
+Bracketing Precision      =  90.53
+Complete match            =  52.94
+Average crossing          =   0.06
+No crossing               =  94.12
+2 or less crossing        = 100.00
+Tagging accuracy          =  98.15
+
+-- len<=40 --
+Number of sentence        =     23
+Number of Error sentence  =      5
+Number of Skip  sentence  =      2
+Number of Valid sentence  =     16
+Bracketing Recall         =  81.25
+Bracketing Precision      =  85.25
+Complete match            =  50.00
+Average crossing          =   0.06
+No crossing               =  93.75
+2 or less crossing        = 100.00
+Tagging accuracy          =  96.88

parsing/EVALB/sample/sample.tst

@@ -0,0 +1,24 @@
+(S (A (P this)) (B (Q is) (A (R a) (T test))))
+(S (A (P this)) (B (Q is) (C (R a) (T test))))
+(S (A (P this)) (B (Q is) (A (R a) (U test))))
+(S (C (P this)) (B (Q is) (A (R a) (U test))))
+(S (A (P this)) (B (Q is) (R a) (A (T test))))
+(S (A (P this) (Q is)) (A (R a) (T test)))
+(S (P this) (Q is) (R a) (T test))
+(P this) (Q is) (R a) (T test)
+(S (A (P this)) (B (Q is) (A (A (R a) (T test)))))
+(S (A (P this)) (B (Q is) (A (A (A (A (A (R a) (T test))))))))
+
+(S (A (P this)) (B (Q was) (A (A (R a) (T test)))))
+(S (A (P this)) (B (Q is) (U not) (A (A (R a) (T test)))))
+
+(TOP (S (A (P this)) (B (Q is) (A (R a) (T test)))))
+(S (A (P this)) (NONE *) (B (Q is) (A (R a) (T test))))
+(S (A (P this)) (S (NONE abc) (A (NONE *))) (B (Q is) (A (R a) (T test))))
+(S (A (P this)) (B (Q is) (A (R a) (TT test))))
+(S (A (P This)) (B (Q is) (A (R a) (T test))))
+(S (A (P That)) (B (Q is) (A (R a) (T test))))
+(S (A (P this)) (B (Q is) (A (R a) (T test))))
+(S (A (P this)) (B (Q is) (A (R a) (T test))) (A (P this)) (B (Q is) (A (R a) (T test))) (A (P this)) (B (Q is) (A (R a) (T test))) (A (P this)) (B (Q is) (A (R a) (T test))) (A (P this)) (B (Q is) (A (R a) (T test))) (A (P this)) (B (Q is) (A (R a) (T test))) (A (P this)) (B (Q is) (A (R a) (T test))) (A (P this)) (B (Q is) (A (R a) (T test))) (A (P this)) (B (Q is) (A (R a) (T test))) (A (P this)) (B (Q is) (A (R a) (T test))) (A (P this)) (B (Q is) (A (R a) (T test))))
+(S (A (P this)) (B (Q is) (A (R a) (T test))) (-NONE- *))
+(S (A (P this)) (B (Q is) (A (R a) (T test))) (: *))

parsing/EVALB/tgrep_proc.prl

@@ -0,0 +1,9 @@
+#!/usr/local/bin/perl
+
+while(<>)
+{
+    if(m/TOP/) #skip lines which are blank
+    {
+	print;
+    } 
+}

parsing/EVALB_SPMRL/Makefile

@@ -0,0 +1,65 @@
+TESTFILE=dev.Arabic.gold.ptb.-feat
+JAVA=java
+
+
+all: evalb
+
+
+clean:
+	rm -f evalb_spmrl
+	
+install: evalb
+	cp evalb_spmrl /usr/local/bin
+
+
+
+evalb: evalb.c
+	gcc -Wall -O3 -g -o evalb_spmrl evalb.c
+	
+evalb_linux: evalb.c
+	gcc -Wall -fPIC -O3 -g -o evalb_spmrl evalb.c
+#to compile on linux
+
+          
+
+# note: on the original makefile, the funsigned-char option was applied
+
+home: install_home
+
+install_home: all
+	cp evalb_spmrl ${PREFIX}/bin
+	
+up:
+	tar zcvf ../evalb_spmrl2013.tar.gz ../evalb_spmrl2013/
+	putW ../evalb_spmrl2013.tar.gz
+
+
+
+#################################
+# stuff to debug some treebanks #
+#################################	
+test_full: all
+	./evalb dev.Arabic.gold.ptb dev.Arabic.gold.ptb
+
+
+test: all
+	./evalb -p ./new.prm ${TESTFILE} ${TESTFILE}
+
+debug: all
+	./evalb -D ${TESTFILE} ${TESTFILE}
+	echo "./evalb -D ${TESTFILE} ${TESTFILE}"
+
+debug_one: all
+	lines 616 < ${TESTFILE} > ${TESTFILE}.616
+	./evalb -D ${TESTFILE}.616 ${TESTFILE}.616
+	echo "./evalb -D ${TESTFILE}.616 ${TESTFILE}.616"	
+
+releaf:
+	./evalb -D dev.Arabic.gold.ptb.-feat.616.bug dev.Arabic.gold.ptb.-feat.616.bug
+	echo "./evalb -D dev.Arabic.gold.ptb.-feat.616.bug dev.Arabic.gold.ptb.-feat.616.bug" > /dev/stderr
+
+java: 
+	${JAVA} -jar ./evalC/evalC.jar ${TESTFILE} ${TESTFILE} /dev/stdout
+		
+	
+		

parsing/EVALB_SPMRL/README

@@ -0,0 +1,76 @@
+
+// Djam�: version record added for history's sake.
+// note to future updater: please add your changelog below
+
+(copied from http://nlp.cs.nyu.edu/evalb/ )
+EVALB20080701.tgz (July 1, 2008 version) modified by Don Blaheta (Knox College)
+EVALB20060307.tgz (March 3, 2006 version; debuged of Jan. 17, 2006 version) modified by David Ellis (Brown University)
+EVALB20060117.tgz (Jan. 17, 2006 version) modified by David Ellis (Brown University)
+EVALB20050908.tgz (Sept. 8, 2005 version) modified by David Brroks (Birmingham)
+EVALB.tgz (original version).
+Authors
+
+Satoshi Sekine (New York University) :  e-mail: his last name (at) cs.nyu.edu
+Michael John Collins (University of Pennsylvania)
+Note: the authors are not responsible for the newer versions. We put these versions even without checking the program. Please be responsible for yourself.
+
+*************************************************************************
+
+Modification
+
+David Brroks (Birmingham): fixed the code so that the program can be compiled by the latest gcc (September 2005). Helps are given by Peet Morris and Ramon Ziai through the Corpora Mailing list.
+David Ellis (Brown University) : fixes a bug in which sentences were incorrectly categorized as "length mismatch" when the the parse output had certain mislabeled parts-of-speech.
+Don Blaheta (KNOX) : fixes a bug on the output of last number of the total information was not TOTAL_crossing, but it was TOTAL_non_crossing.
+
+
+
+April 2012
+// Modified by Slav Petrov and Ryanc Mc Donald (Google inc., for  the sancl 2012 shared task)
+// ===>  making it less sensitive to punct POS errors leading to
+// mismatch of length
+
+
+August 2013, 10
+// Modified by Djam� Seddah (Univ. Paris Sorbonne, for the spmrl 2013 shared  task)
+// ===> making it able to cope with Arabic very long lines (byte wise)
+// ===>  now limit is 50000 bytes, was 5000 (tricky bug, if you ask me)
+// please check the constant macro section if you encounter weird bugs not present in other
+implementations (check evalC by Federico Sangatti for example, http://homepages.inf.ed.ac.uk/fsangati/evalC_25_5_10.zip )
+
+
+August 2013, 23
+// Modif from Thomas M�ller (IMS Stuttgart)
+// ===> adding of # in the stop word modify_label function (so that the
+// lexer will read NPP instead of NPP##feat:...### as in hte SPMRL Data set
+// Modif from Djam� Seddah
+// ===>  Application of modify_label to all labels (including the POS label
+// wich were left untouched for some reasons)
+// That should btw be an option. (wether to evaluate full labels or not,
+// only stripping of Non Terminal, POS tag and so on)
+ 
+
+August 2013, 27
+// Modif from Djam�
+// --> adding of an option to include the non parsed sentences in the
+// --> evaluation (-X option)
+// --> adding an option to evaluate only the first N sentences (-K n)
+// --> adding an option to provide a compact results view (-L) so one can do
+// --> find ./ -name "*parsed.run?" -exec evalb_spmrl -L GOLD {} \; -print |
+// --> grep -v '=====' | grep '='
+
+September 2013, 6
+// Modif from DJame
+// fixing the infinite slowness bug (shame on me)
+// now speed is similar to what it was before
+
+
+October 2013, 13
+// Addition from Djame
+// Adding the spmrl_hebrew.prm if one wants to evaluate hebrew parsing within the
+// same conditions as the state-of-the-art
+// namely without counting the additional SYNpos layer which inflates evalb
+// scores by almost 2 points.
+// Note: for the spmrl shared task, we used the spmrl.prm file (so with
+// these labels. It was too late to modify the rules once again when we
+// realized this)
+

parsing/EVALB_SPMRL/README.orig

@@ -0,0 +1,230 @@
+#################################################################
+#                                                               #
+#      README file for evalb                                    #
+#                                                               #
+#                                         Satoshi Sekine (NYU)  #
+#                                         Mike Collins (UPenn)  #
+#                                                               #
+#                                         October.1997          #
+#################################################################
+
+Contents of this README:
+
+   [0] COPYRIGHT
+   [1] INTRODUCTION
+   [2] INSTALLATION AND RUN
+   [3] OPTIONS
+   [4] OUTPUT FORMAT FROM THE SCORER
+   [5] HOW TO CREATE A GOLDFILE FROM THE TREEBANK
+   [6] THE PARAMETER FILE
+   [7] MORE DETAILS ABOUT THE SCORING ALGORITHM
+
+
+[0] COPYRIGHT
+
+The authors abandon the copyright of this program. Everyone is 
+permitted to copy and distribute the program or a portion of the program
+with no charge and no restrictions unless it is harmful to someone.
+
+However, the authors are delightful for the user's kindness of proper
+usage and letting the authors know bugs or problems.
+
+This software is provided "AS IS", and the authors make no warranties,
+express or implied.
+
+
+[1] INTRODUCTION
+
+Evaluation of bracketing looks simple, but in fact, there are minor
+differences from system to system. This is a program to parametarize
+such minor differences and to give an informative result.
+
+"evalb" evaluates bracketing accuracy in a test-file against a gold-file.
+It returns recall, precision, tagging accuracy. It uses an identical 
+algorithm to that used in (Collins ACL97).
+
+
+[2] Installation and Run
+
+To compile the scorer, type 
+
+> make
+
+
+To run the scorer:
+
+> evalb -p Parameter_file Gold_file Test_file
+
+ 
+For example to use the sample files:
+
+> evalb -p sample.prm sample.gld sample.tst
+
+
+
+[3] OPTIONS
+
+You can specify system parameters in the command line options.
+Other options concerning to evaluation metrix should be specified
+in parameter file, described later.
+
+        -p param_file  parameter file                        
+        -d             debug mode                            
+        -e n           number of error to kill (default=10)  
+        -h             help                                  
+
+
+
+[4] OUTPUT FORMAT FROM THE SCORER
+
+The scorer gives individual scores for each sentence, for
+example:
+
+  Sent.                        Matched  Bracket   Cross        Correct Tag
+ ID  Len.  Stat. Recal  Prec.  Bracket gold test Bracket Words  Tags Accracy
+============================================================================
+   1    8    0  100.00 100.00     5      5    5      0      6     5    83.33
+
+At the end of the output the === Summary === section gives statistics 
+for all sentences, and for sentences <=40 words in length. The summary
+contains the following information:
+
+i)   Number of sentences -- total number of sentences.
+
+ii)  Number of Error/Skip sentences -- should both be 0 if there is no
+    problem with the parsed/gold files.
+
+iii) Number of valid sentences = Number of sentences - Number of Error/Skip
+    sentences 
+
+iv)  Bracketing recall =     (number of correct constituents)
+                         ----------------------------------------
+                         (number of constituents in the goldfile)
+
+v)   Bracketing precision = (number of correct constituents)
+                         ----------------------------------------
+                         (number of constituents in the parsed file)
+
+vi)  Complete match = percentaage of sentences where recall and precision are
+    both 100%. 
+
+vii) Average crossing = (number of constituents crossing a goldfile constituen
+                         ----------------------------------------------------
+                                        (number of sentences)
+
+viii) No crossing = percentage of sentences which have 0 crossing brackets.
+
+ix)   2 or less crossing = percentage of sentences which have <=2 crossing brackets.
+
+x)    Tagging accuracy = percentage of correct POS tags (but see [5].3 for exact
+     details of what is counted).
+
+
+
+[5] HOW TO CREATE A GOLDFILE FROM THE PENN TREEBANK
+
+
+The gold and parsed files are in a format similar to this:
+
+(TOP (S (INTJ (RB No)) (, ,) (NP (PRP it)) (VP (VBD was) (RB n't) (NP (NNP Black) (NNP Monday))) (. .)))
+
+To create a gold file from the treebank:
+
+tgrep -wn '/.*/' | tgrep_proc.prl 
+
+will produce a goldfile in the required format.  ("tgrep -wn '/.*/'" prints
+parse trees, "tgrep_process.prl" just skips blank lines).
+
+For example, to produce a goldfile for section 23 of the treebank:
+
+tgrep -wn '/.*/' | tail +90895 | tgrep_process.prl | sed 2416q > sec23.gold
+
+
+
+[6] THE PARAMETER (.prm) FILE
+
+
+The .prm file sets options regarding the scoring method. COLLINS.prm gives
+the same scoring behaviour as the scorer used in (Collins 97). The options 
+chosen were: 
+
+1) LABELED 1
+
+to give labelled precision/recall figures, i.e. a constituent must have the
+same span *and* label as a constituent in the goldfile.
+
+2) DELETE_LABEL TOP   
+
+Don't count the "TOP" label (which is always given in the output of tgrep) 
+when scoring. 
+
+3) DELETE_LABEL -NONE-  
+
+Remove traces (and all constituents which dominate nothing but traces) when
+scoring. For example
+
+.... (VP (VBD reported) (SBAR (-NONE- 0) (S (-NONE- *T*-1)))) (. .)))
+
+would be processed to give
+
+.... (VP (VBD reported)) (. .)))
+
+
+4)
+DELETE_LABEL ,     -- for the purposes of scoring remove punctuation
+DELETE_LABEL :
+DELETE_LABEL ``
+DELETE_LABEL ''
+DELETE_LABEL .
+
+5) DELETE_LABEL_FOR_LENGTH -NONE-   -- don't include traces when calculating
+                                       the length of a sentence (important
+                                       when classifying a sentence as <=40
+                                       words or >40 words)
+
+6) EQ_LABEL ADVP PRT
+
+Count ADVP and PRT as being the same label when scoring.
+
+
+
+
+[7] MORE DETAILS ABOUT THE SCORING ALGORITHM
+
+
+1) The scorer initially processes the files to remove all nodes specified
+by DELETE_LABEL in the .prm file. It also recursively removes nodes which
+dominate nothing due to all their children being removed. For example, if
+-NONE- is specified as a label to be deleted, 
+
+.... (VP (VBD reported) (SBAR (-NONE- 0) (S (-NONE- *T*-1)))) (. .)))
+
+would be processed to give
+
+.... (VP (VBD reported)) (. .)))
+
+2) The scorer also removes all functional tags attached to non-terminals
+(functional tags are prefixed with "-" or "=" in the treebank). For example
+"NP-SBJ" is processed to give "NP", "NP=2" is changed to "NP".
+
+
+3) Tagging accuracy counts tags for all words *except* any tags which are
+deleted by a DELETE_LABEL specification in the .prm file. (For example, for
+COLLINS.prm, punctuation tagged as "," ":" etc. would not be included).
+
+4) When calculating the length of a sentence, all words with POS tags not 
+included in the "DELETE_LABEL_FOR_LENGTH" list in the .prm file are
+counted. (For COLLINS.prm, only "-NONE-" is specified in this list, so
+traces are removed before calculating the length of the sentence).
+
+5) There are some subtleties in scoring when either the goldfile or parsed
+file contains multiple constituents for the same span which have the same
+non-terminal label. e.g. (NP (NP the man)) If the goldfile contains n 
+constituents for the same span, and the parsed file contains m constituents
+with that nonterminal, the scorer works as follows:
+
+i) If m>n, then the precision is n/m, recall is 100%
+
+ii) If n>m, then the precision is 100%, recall is m/n.
+
+iii) If n==m, recall and precision are both 100%.

parsing/EVALB_SPMRL/evalb.c

@@ -0,0 +1,1724 @@
+/*****************************************************************/
+/* evalb [-p param_file] [-dh] [-e n] gold-file test-file        */
+/*                                                               */
+/*        Evaluate bracketing in test-file against gold-file.    */
+/*        Return recall, precision, tagging accuracy.            */
+/*                                                               */
+/*   <option>                                                    */
+/*        -p param_file  parameter file                          */
+/*        -d             debug mode                              */
+/*        -e n           number of error to kill (default=10)    */
+/*        -h             help                                    */
+/*                                                               */
+/*                                         Satoshi Sekine (NYU)  */
+/*                                         Mike Collins (UPenn)  */
+/*                                                               */
+/*                                         October.1997          */
+/*                                                               */
+/* Please refer README for the update information                */
+/*****************************************************************/
+
+// Djamé: version record added for history's sake.
+// note to future updater: please add your changelog below
+
+// Modified by Slav Petrov and Ryanc Mc Donald (for  the sancl 2012 shared task)
+// ===>  making it less sensitive to punct POS errors leading to
+// ===> mismatch of length
+
+// Modified by Djamé Seddah (for the spmrl shared 2013 task)
+// ===> making it able to cope with Arabic very long lines (byte wise)
+// ===>  now limit is 50000 bytes, was 5000 (damn bug, if you ask me)
+// ===> modified to cope with spmrl 2013 morpg features (suggested by thomas Muller from IMS)
+// please check the constant macro section
+// Correction of bug causing hard slowdown (due to max_word_in_sent set too high)
+// former version was 78x slower than regular evalb.
+
+
+
+
+
+
+#include <stdio.h>
+#include <stdlib.h> //### added for exit, atoi decls
+#include <ctype.h>
+#include <string.h>
+#ifndef __APPLE__  # dj: added to compile on mac os x
+#include <malloc.h>
+#endif
+
+/* Internal Data format -------------------------------------------*/
+/*                                                                 */
+/* (S (NP (NNX this)) (VP (VBX is) (NP (DT a) (NNX pen))) (SYM .)) */
+/*                                                                 */
+/*   wn=5                                                          */
+/*                        word    label                            */
+/*   terminal[0] =        this     NNX                             */
+/*   terminal[1] =        is       VBX                             */
+/*   terminal[2] =        a        DT                              */
+/*   terminal[3] =        pen      NNX                             */
+/*   terminal[4] =        .        SYM                             */
+/*                                                                 */
+/*   bn=4                                                          */
+/*                      start     end      label                   */
+/*   bracket[0]  =        0        5         S                     */
+/*   bracket[1]  =        0        0         NP                    */
+/*   bracket[2]  =        1        4         VP                    */
+/*   bracket[3]  =        2        4         NP                    */
+/*                                                                 */
+/*              matched bracketing                                 */
+/*   Recall = ---------------------------                          */
+/*             # of bracket in ref-data                            */
+/*                                                                 */
+/*              matched bracketing                                 */
+/*   Recall = ---------------------------                          */
+/*             # of bracket in test-data                           */
+/*                                                                 */
+/*-----------------------------------------------------------------*/
+
+/******************/
+/* constant macro */
+/******************/
+
+#define MAX_SENT_LEN            50000   //Djamé : was not used
+#define MAX_WORD_IN_SENT        1000
+#define MAX_BRACKET_IN_SENT     2000
+#define MAX_WORD_LEN            100
+#define MAX_LABEL_LEN           300
+#define MAX_QUOTE_TERM           20
+
+#define MAX_DELETE_LABEL        1000
+#define MAX_EQ_LABEL            1000
+#define MAX_EQ_WORD             1000
+
+#define MAX_LINE_LEN            500
+
+#define DEFAULT_MAX_ERROR        10
+#define DEFAULT_CUT_LEN          40
+
+/*************/
+/* structure */
+/*************/
+
+typedef struct ss_terminal {
+    char word[MAX_WORD_LEN];
+    char label[MAX_LABEL_LEN];
+    int  result;                /* 0:unmatch, 1:match, 9:undef */
+} s_terminal;
+
+typedef struct ss_term_ind {
+	s_terminal term;
+	int index;
+	int bracket;
+	int endslen;
+	int ends[MAX_BRACKET_IN_SENT];
+} s_term_ind;
+
+typedef struct ss_bracket {
+    int start;
+    int end;
+    unsigned int buf_start;
+    unsigned int buf_end;
+    char label[MAX_LABEL_LEN];
+    int  result;                 /* 0: unmatch, 1:match, 5:delete 9:undef */
+} s_bracket;
+
+
+typedef struct ss_equiv {
+    char *s1;
+    char *s2;
+} s_equiv;
+
+
+/****************************/
+/* global variables         */
+/*   gold-data: suffix = 1  */
+/*   test-data: suffix = 2  */
+/****************************/
+
+/*---------------*/
+/* Sentence data */
+/*---------------*/
+int wn1, wn2;                              /* number of words in sentence  */
+int r_wn1;                                 /* number of words in sentence  */
+/* which only ignores labels in */
+/* DELETE_LABEL_FOR_LENGTH      */
+
+s_terminal terminal1[MAX_WORD_IN_SENT];    /* terminal information */
+s_terminal terminal2[MAX_WORD_IN_SENT];
+
+s_term_ind quotterm1[MAX_QUOTE_TERM];      /* special terminals ("'","POS") */
+s_term_ind quotterm2[MAX_QUOTE_TERM];
+
+int bn1, bn2;                              /* number of brackets */
+
+int r_bn1, r_bn2;                          /* number of brackets */
+/* after deletion */
+
+s_bracket bracket1[MAX_BRACKET_IN_SENT];   /* bracket information */
+s_bracket bracket2[MAX_BRACKET_IN_SENT];
+
+
+/*------------*/
+/* Total data */
+/*------------*/
+int TOTAL_bn1, TOTAL_bn2, TOTAL_match;     /* total number of brackets */
+int TOTAL_sent;                            /* No. of sentence */
+int TOTAL_error_sent;                      /* No. of error sentence */
+int TOTAL_skip_sent;                       /* No. of skip sentence */
+int TOTAL_comp_sent;                       /* No. of complete match sent */
+int TOTAL_word;                            /* total number of word */
+int TOTAL_crossing;                        /* total crossing */
+int TOTAL_no_crossing;                     /* no crossing sentence */
+int TOTAL_2L_crossing;                     /* 2 or less crossing sentence */
+int TOTAL_correct_tag;                     /* total correct tagging */
+
+int TOT_cut_len = DEFAULT_CUT_LEN;         /* Cut-off length in statistics */
+
+/* data for sentences with len <= CUT_LEN */
+/* Historically it was 40.                */
+int TOT40_bn1, TOT40_bn2, TOT40_match;     /* total number of brackets */
+int TOT40_sent;                            /* No. of sentence */
+int TOT40_error_sent;                      /* No. of error sentence */
+int TOT40_skip_sent;                       /* No. of skip sentence */
+int TOT40_comp_sent;                       /* No. of complete match sent */
+int TOT40_word;                            /* total number of word */
+int TOT40_crossing;                        /* total crossing */
+int TOT40_no_crossing;                     /* no crossing sentence */
+int TOT40_2L_crossing;                     /* 2 or less crossing sentence */
+int TOT40_correct_tag;                     /* total correct tagging */
+
+/*------------*/
+/* miscallous */
+/*------------*/
+int Line;                                  /* line number */
+int Error_count = 0;                       /* Error count */
+int Status;                                /* Result status for each sent */
+/*    0: OK, 1: skip, 2: error */
+
+/*-------------------*/
+/* stack manuplation */
+/*-------------------*/
+int stack_top;
+int stack[MAX_BRACKET_IN_SENT];
+
+/************************************************************/
+/* User parameters which can be specified in parameter file */
+/************************************************************/
+
+/*------------------------------------------*/
+/* Debug mode                               */
+/*   print out data for individual sentence */
+/*------------------------------------------*/
+int DEBUG=0;
+
+/*------------------------------------------*/
+/* MAX error                                */
+/*    Number of error to stop the process.  */
+/*    This is useful if there could be      */
+/*    tokanization error.                   */
+/*    The process will stop when this number*/
+/*    of errors are accumulated.            */
+/*------------------------------------------*/
+int Max_error = DEFAULT_MAX_ERROR;
+
+/*------------------------------------------*/
+/* Cut-off length for statistics            */
+/*    int TOT_cut_len = DEFAULT_CUT_LEN;    */
+/*    (Defined above)                       */
+/*------------------------------------------*/
+
+
+/*------------------------------------------*/
+/* unlabeled or labeled bracketing          */
+/*    0: unlabeled bracketing               */
+/*    1: labeled bracketing                 */
+/*------------------------------------------*/
+int F_label    = 1;                 
+
+/*------------------------------------------*/
+/* Delete labels                            */
+/*    list of labels to be ignored.         */
+/*    If it is a pre-terminal label, delete */
+/*    the word along with the brackets.     */
+/*    If it is a non-terminal label, just   */
+/*    delete the brackets (don't delete     */
+/*    childrens).                           */
+/*------------------------------------------*/
+char *Delete_label[MAX_DELETE_LABEL];
+int Delete_label_n = 0;
+
+/*------------------------------------------*/
+/* Delete labels for length calculation     */
+/*    list of labels to be ignored for      */
+/*    length calculation purpose            */
+/*------------------------------------------*/
+char *Delete_label_for_length[MAX_DELETE_LABEL];
+int Delete_label_for_length_n = 0;
+
+/*------------------------------------------*/
+/* Labels to be considered for misquote     */
+/*    (could be possesive or quote)         */
+/*------------------------------------------*/
+char *Quote_term[MAX_QUOTE_TERM];
+int Quote_term_n = 0;
+
+/*------------------------------------------*/
+/* Equivalent labels, words                 */
+/*     the pairs are considered equivalent  */
+/*     This is non-directional.             */
+/*------------------------------------------*/
+s_equiv EQ_label[MAX_EQ_LABEL];
+int EQ_label_n = 0;
+
+s_equiv EQ_word[MAX_EQ_WORD];
+int EQ_word_n = 0;
+
+
+// added by djame
+int spmrl_max_line_to_read=-1	;
+int spmrl_compact_view=0; // default : classic view
+int spmrl_compact_view40=0; // if one, prints <40 sentence in compact view
+int spmrl_count_bad_sent=0; // default no count
+int spmrl_print_filename=0; // default not to print name
+
+/************************/
+/* Function return-type */
+/************************/
+int main();
+void init_global();
+void print_head();
+void init();
+void read_parameter_file();
+void set_param();
+int narg();
+int read_line();
+
+void pushb();
+int popb();
+int stackempty();
+
+void calc_result(unsigned char *buf1,unsigned char *buf);
+void fix_quote();
+void reinsert_term();
+void massage_data();
+int massage_data_gold_only(); // djame: non destructive 
+void modify_label();
+void individual_result();
+void print_total();
+void dsp_info();
+int my_isspace(char c); // Djamé: added for debugging' sake
+
+int is_terminator();
+int is_deletelabel();
+int is_deletelabel_for_length();
+int is_quote_term();
+int word_comp();
+int label_comp();
+
+void Error();
+void Fatal();
+void Usage();
+
+/* ### provided by std headers 
+ int fprintf();
+ int printf();
+ int atoi();
+ int fclose();
+ int sscanf();
+ */
+
+/***********/
+/* program */
+/***********/
+#define ARG_CHECK(st) if(!(*++(*argv) || (--argc && *++argv))){ \
+fprintf(stderr,"Missing argument: %s\n",st); \
+}
+
+
+    char *filename1, *filename2;
+int
+main(argc,argv)
+int argc;
+char *argv[];
+{
+
+    FILE *fd1, *fd2;
+    unsigned char buff[MAX_SENT_LEN];
+    unsigned char buff1[MAX_SENT_LEN];
+	int quiet=0; // Djame
+	filename1=NULL;
+	filename2=NULL;
+	
+	
+    for(argc--,argv++;argc>0;argc--,argv++){
+		if(**argv == '-'){
+			while(*++(*argv)){
+				switch(**argv){
+						
+					case 'h':    /* help */
+						Usage();
+						exit(1);
+						
+					case 'd':      /* debug mode */
+						DEBUG = 1;
+						goto nextarg;
+						
+					case 'D':      /* debug mode */
+						DEBUG = 2;
+						goto nextarg;
+						
+					case 'c':      /* cut-off length */
+						ARG_CHECK("cut-off length for statistices");
+						TOT_cut_len = atoi(*argv);
+						fprintf(stderr,"cutoff %d\n",TOT_cut_len);
+						//exit(0);
+						goto nextarg;
+						
+
+					case 'e':      /* max error */
+						ARG_CHECK("number of error to kill");
+						Max_error = atoi(*argv);
+						goto nextarg;
+						
+					case 'p':      /* parameter file */
+						ARG_CHECK("parameter file");
+						read_parameter_file(*argv);
+						goto nextarg;
+					case 'K':
+						ARG_CHECK("Max nb of  sentences to read");
+						spmrl_max_line_to_read=atoi(*argv);
+						goto nextarg;
+					case 'L':  // added by djame to maintain compatibility with spmrl 2013 shared task's results extraction rules. 
+						spmrl_compact_view=1;
+						goto nextarg;
+					case 'l':  // added by djame to maintain compatibility with spmrl 2013 shared task's results extraction rules. 
+						spmrl_compact_view=1;
+						spmrl_compact_view40=1;
+						goto nextarg;
+					case 'X': // added by djame : count skipping sentences (()) as bad sentence 
+						spmrl_count_bad_sent=1;
+						goto nextarg;
+					case 'V': // added by djame to add gold_name vs test_file in the outpu
+						spmrl_print_filename=1;
+						goto nextarg;
+					default:
+						Usage();
+						exit(0);
+				}
+			}
+		} else {
+			if(filename1==NULL){
+				filename1 = *argv;
+			}else if(filename2==NULL){
+				filename2 = *argv;
+			}
+		}
+	nextarg: continue;
+    }
+	
+    init_global();
+	
+	
+    if((fd1 = fopen(filename1,"r"))==NULL){
+		Fatal("Can't open gold file (%s)\n",filename1);
+    }
+    if((fd2 = fopen(filename2,"r"))==NULL){
+		Fatal("Can't open test file (%s)\n",filename2);
+    }
+	
+    print_head();
+	
+    for(Line=1;fgets(buff,MAX_SENT_LEN,fd1)!=NULL;Line++){
+		
+		init();
+		
+		/* READ 1 */
+		r_wn1 = read_line(buff,terminal1,quotterm1,&wn1,bracket1,&bn1);
+		
+		strcpy(buff1,buff);
+		
+		/* READ 2 */
+		if(fgets(buff,MAX_SENT_LEN,fd2)==NULL){
+			Error("Number of lines unmatch (too many lines in gold file)\n");
+			break;
+		}
+		
+		read_line(buff,terminal2,quotterm2,&wn2,bracket2,&bn2);
+				
+		/* Calculate result and print it */
+		calc_result(buff1,buff);
+		
+		if(DEBUG>=1){
+			dsp_info();
+		}
+		// Added by djame
+		if (spmrl_max_line_to_read!=-1){
+			if ((Line+1) > spmrl_max_line_to_read ){
+				quiet=1;
+				break; // evaluate only  spmrl_max_line_to_read -1 (to keep compatibility with lines )
+			}
+		}
+
+    }
+	
+    if( (quiet==0) && (fgets(buff,MAX_SENT_LEN,fd2)!=NULL)){
+		Error("Number of lines unmatch (too many lines in test file)\n");
+    }
+	
+    print_total();
+	
+    return (0);
+}
+
+
+/*-----------------------------*/
+/* initialize global variables */
+/*-----------------------------*/
+void
+init_global()
+{
+    TOTAL_bn1 = TOTAL_bn2 = TOTAL_match = 0;
+    TOTAL_sent = TOTAL_error_sent = TOTAL_skip_sent = TOTAL_comp_sent = 0;
+    TOTAL_word = TOTAL_correct_tag = 0;
+    TOTAL_crossing = 0;
+    TOTAL_no_crossing = TOTAL_2L_crossing = 0;
+	
+    TOT40_bn1 = TOT40_bn2 = TOT40_match = 0;
+    TOT40_sent = TOT40_error_sent = TOT40_skip_sent = TOT40_comp_sent = 0;
+    TOT40_word = TOT40_correct_tag = 0;
+    TOT40_crossing = 0;
+    TOT40_no_crossing = TOT40_2L_crossing = 0;
+	
+}
+
+
+/*------------------*/
+/* print head title */
+/*------------------*/
+void
+print_head()
+{
+    printf("  Sent.                        Matched  Bracket   Cross        Correct Tag\n");
+    printf(" ID  Len.  Stat. Recal  Prec.  Bracket gold test Bracket Words  Tags Accracy\n");
+    printf("============================================================================\n");
+}
+
+
+/*-----------------------------------------------*/
+/* initialization at each individual computation */
+/*-----------------------------------------------*/
+void
+init()
+{
+	int i;
+	
+	wn1 = 0;
+	wn2 = 0;
+	bn1 = 0;
+	bn2 = 0;
+	r_bn1 = 0;
+	r_bn2 = 0;
+	
+	for(i=0;i<MAX_WORD_IN_SENT;i++){
+		terminal1[i].word[0]  = '\0';
+		terminal1[i].label[0] = '\0';
+		terminal1[i].result   = 9;
+		terminal2[i].word[0]  = '\0';
+		terminal2[i].label[0] = '\0';
+		terminal2[i].result   = 9;
+	}
+	
+	for(i=0;i<MAX_QUOTE_TERM;i++){
+		quotterm1[i].term.word[0]  = '\0';
+		quotterm1[i].term.label[0] = '\0';
+		quotterm1[i].term.result   = 9;
+		quotterm1[i].index         = -1;
+		quotterm1[i].bracket       = -1;
+		quotterm2[i].term.word[0]  = '\0';
+		quotterm2[i].term.label[0] = '\0';
+		quotterm2[i].term.result   = 9;
+		quotterm2[i].index         = -1;
+		quotterm2[i].bracket       = -1;
+	}
+	
+	for(i=0;i<MAX_BRACKET_IN_SENT;i++){
+		bracket1[i].start    = -1;
+		bracket1[i].end      = -1;
+		bracket1[i].label[0] = '\0';
+		bracket1[i].result   = 9;
+		bracket2[i].start    = -1;
+		bracket2[i].end      = -1;
+		bracket2[i].label[0] = '\0';
+		bracket2[i].result   = 9;
+	}
+	
+	Status = 0;
+}
+
+/*----------------*/
+/* parameter file */
+/*----------------*/
+void
+read_parameter_file(filename)
+char *filename;
+{
+    char buff[MAX_LINE_LEN];
+    FILE *fd;
+    int line;
+    int i;
+	
+    if((fd=fopen(filename,"r"))==NULL){
+		Fatal("Can't open parameter file (%s)\n",filename);
+    }
+	
+    for(line=1;fgets(buff,MAX_LINE_LEN,fd)!=NULL;line++){
+		
+		/* clean up the tail and find unvalid line */
+		/*-----------------------------------------*/
+		for(i=strlen(buff)-1;i>0 && (isspace(buff[i]) || buff[i]=='\n');i--){
+			buff[i]='\0';
+		}
+		if(buff[0]=='#' ||      /* comment-line */
+		   strlen(buff)<3){     /* too short, just ignore */
+			continue;
+		}
+		
+		/* place the parameter and value */
+		/*-------------------------------*/
+		for(i=0;!isspace(buff[i]);i++);
+		for(;isspace(buff[i]) && buff[i]!='\0';i++);
+		if(buff[i]=='\0'){
+			fprintf(stderr,"Empty value in parameter file (%d)\n",line);
+		}
+		
+		/* set parameter and value */
+		/*-------------------------*/
+		set_param(buff,buff+i);
+    }
+	
+    fclose(fd);
+}
+
+
+#define STRNCMP(s) (strncmp(param,s,strlen(s))==0 &&  \
+(param[strlen(s)]=='\0' || isspace(param[strlen(s)])))
+
+
+void
+set_param(param,value)
+char *param, *value;
+{
+    char l1[MAX_LABEL_LEN], l2[MAX_LABEL_LEN];
+	
+    if(STRNCMP("DEBUG")){
+		
+		DEBUG = atoi(value);
+		
+    }else if(STRNCMP("MAX_ERROR")){
+		
+		Max_error = atoi(value);
+		
+    }else if(STRNCMP("CUTOFF_LEN")){
+		
+		TOT_cut_len = atoi(value);
+		
+    }else if(STRNCMP("LABELED")){
+		
+		F_label = atoi(value);
+		
+    }else if(STRNCMP("DELETE_LABEL")){
+		
+		Delete_label[Delete_label_n] = (char *)malloc(strlen(value)+1);
+		strcpy(Delete_label[Delete_label_n],value);
+		Delete_label_n++;
+		
+    }else if(STRNCMP("DELETE_LABEL_FOR_LENGTH")){
+		
+		Delete_label_for_length[Delete_label_for_length_n] = (char *)malloc(strlen(value)+1);
+		strcpy(Delete_label_for_length[Delete_label_for_length_n],value);
+		Delete_label_for_length_n++;
+		
+    }else if(STRNCMP("QUOTE_LABEL")){
+		
+		Quote_term[Quote_term_n] = (char *)malloc(strlen(value)+1);
+		strcpy(Quote_term[Quote_term_n],value);
+		Quote_term_n++;
+		
+    }else if(STRNCMP("EQ_LABEL")){
+		
+		if(narg(value)!=2){
+			fprintf(stderr,"EQ_LABEL requires two values\n");
+			return;
+		}
+		sscanf(value,"%s %s",l1,l2);
+		EQ_label[EQ_label_n].s1 = (char *)malloc(strlen(l1)+1);
+		strcpy(EQ_label[EQ_label_n].s1,l1);
+		EQ_label[EQ_label_n].s2 = (char *)malloc(strlen(l2)+1);
+		strcpy(EQ_label[EQ_label_n].s2,l2);
+		EQ_label_n++;
+		
+    }else if(STRNCMP("EQ_WORD")){
+		
+		if(narg(value)!=2){
+			fprintf(stderr,"EQ_WORD requires two values\n");
+			return;
+		}
+		sscanf(value,"%s %s",l1,l2);
+		EQ_word[EQ_word_n].s1 = (char *)malloc(strlen(l1)+1);
+		strcpy(EQ_word[EQ_word_n].s1,l1);
+		EQ_word[EQ_word_n].s2 = (char *)malloc(strlen(l2)+1);
+		strcpy(EQ_word[EQ_word_n].s2,l2);
+		EQ_word_n++;
+		
+    }else{
+		
+		fprintf(stderr,"Unknown keyword (%s) in parameter file\n",param);
+		
+    }
+}
+
+
+int
+narg(s)
+char *s;
+{
+    int n;
+	
+    for(n=0;*s!='\0';){
+		for(;isspace(*s);s++);
+		if(*s=='\0'){
+			break;
+		}
+		n++;
+		for(;!isspace(*s);s++){
+			if(*s=='\0'){
+				break;
+			}
+		}
+    }
+	
+    return(n);
+}
+
+/*-----------------------------*/
+/* Read line and gather data.  */
+/* Return langth of sentence.  */
+/*-----------------------------*/
+int
+read_line(buff, terminal, quotterm, wn, bracket, bn)
+char *buff;
+s_terminal terminal[];
+s_term_ind quotterm[];
+int *wn;
+s_bracket bracket[];
+int *bn;
+{
+    char *p, *q, label[MAX_LABEL_LEN], word[MAX_WORD_LEN];
+    int	  qt;		  /* quote term counter */
+    int   wid, bid;       /* word ID, bracket ID */
+    int   n;              /* temporary remembering the position */
+    int   b;              /* temporary remembering bid */
+    int   i;
+    int   len;            /* length of the sentence */
+	
+    len = 0;
+    stack_top=0;
+	
+    for(p=buff,qt=0,wid=0,bid=0;*p!='\0';){
+		
+		if(isspace(*p)){
+			p++;
+			continue;
+			
+			/* open bracket */
+			/*--------------*/
+		}else if(*p=='('){
+			
+			n=wid;
+			for(p++,i=0;!is_terminator(*p);p++,i++){
+				label[i]=*p;
+			}
+			label[i]='\0';
+			
+			/* Find terminals */
+			q = p;
+			if(isspace(*q)){
+				for(q++;isspace(*q);q++);
+				for(i=0;!is_terminator(*q);q++,i++){
+					word[i]=*q;
+				}
+				word[i]='\0';
+				
+                /* compute length */
+				if(*q==')' && !is_deletelabel_for_length(label)==1){
+					len++;
+				}
+				if (DEBUG>1)
+					printf("label=%s, word=%s, wid=%d\n",label,word,wid);
+				/* quote terminal */
+				if(*q==')' && is_quote_term(label,word)==1){
+					strcpy(quotterm[qt].term.word,word);
+					strcpy(quotterm[qt].term.label,label);
+					quotterm[qt].index = wid;
+					quotterm[qt].bracket = bid;
+					quotterm[qt].endslen = stack_top;
+					//quotterm[qt].ends = (int*)malloc(stack_top*sizeof(int));
+					memcpy(quotterm[qt].ends,stack,stack_top*sizeof(int));
+					qt++;
+				}
+				
+				/* Slav: do not delete terminals */
+                /* delete terminal */
+				//if(*q==')' && is_deletelabel(label)==1){
+				//    p = q+1;
+				//    continue;
+				
+				/* valid terminal */
+				//}else 
+				if(*q==')'){
+					strcpy(terminal[wid].word,word);
+					strcpy(terminal[wid].label,label);
+					wid++;
+					p = q+1;
+					continue;
+					
+					/* error */
+				}else if(*q!='('){
+					fprintf(stderr,"debug djam: q= %s\n",q);
+					Error("More than two elements in a bracket\n");
+				}
+			}
+			
+            /* otherwise non-terminal label */
+			bracket[bid].start = wid;
+			bracket[bid].buf_start = p-buff;
+			strcpy(bracket[bid].label,label);
+			pushb(bid);
+			bid++;
+			
+			/* close bracket */
+			/*---------------*/
+		}else if(*p==')'){
+			
+			b = popb();
+			bracket[b].end = wid;
+			bracket[b].buf_end = p-buff;
+			p++;
+			
+			/* error */
+			/*-------*/
+		}else{
+			
+			Error("Reading sentence\n");
+		}
+    }
+	
+    if(!stackempty()){
+		Error("Bracketing is unbalanced (too many open bracket)\n");
+    }
+	
+    *wn = wid;
+    *bn = bid;
+	
+    return(len);
+}
+
+
+/*----------------------*/
+/* stack operation      */
+/* for bracketing pairs */
+/*----------------------*/
+void
+pushb(item)
+int item;
+{
+    stack[stack_top++]=item;
+}
+
+int
+popb()
+{
+    int item;
+	
+    item = stack[stack_top-1];
+	
+    if(stack_top-- < 0){
+		Error("Bracketing unbalance (too many close bracket)\n");
+    }
+    return(item);
+}
+
+int
+stackempty()
+{
+    if(stack_top==0){
+		return(1);
+    }else{
+		return(0);
+    }
+}
+
+
+/*------------------*/
+/* calculate result */
+/*------------------*/
+void
+calc_result(unsigned char *buf1,unsigned char *buf)
+{
+    int i, j, l;
+    int match, crossing, correct_tag;
+	
+    int last_i = -1;
+	
+    char my_buf[10000]; //djame: was 1000
+    int match_found = 0;
+    
+    char match_j[2000]; //djame was : 200
+    for (j = 0; j < bn2; ++j) {
+		match_j[j] = 0;
+    }
+	
+    /* ML */
+    if (DEBUG>1)
+    	printf("\n");
+	
+	
+    /* Find skip and error */
+    /*---------------------*/
+    if(wn2==0){ // Djame: case of empty lines
+		if (spmrl_count_bad_sent==1){
+			Status = 3; 
+			//individual_result(wn1,r_bn1,r_bn2,match,crossing,correct_tag);
+			int n_bracket_gold=massage_data_gold_only();
+			r_bn1=n_bracket_gold;
+			individual_result(wn1,n_bracket_gold,0,0,0,0); // testing the case of missing analysis was 0,0
+		}else {
+			Status=2;
+			individual_result(0,0,0,0,0,0);
+		}
+
+		return;
+    }
+	
+	if(wn1 != wn2){
+		//if (DEBUG>1)
+		//Error("Length unmatch (%d|%d)\n",wn1,wn2);
+		fix_quote();
+		if(wn1 != wn2){
+			individual_result(0,0,0,0,0,0); 
+			/* Slav: ignore 1 word sentences */
+			if (wn1 > 1) {
+				Error("Length unmatch (%d|%d)\n",wn1,wn2);
+				return;
+			}
+		}
+    }
+	
+    for(i=0;i<wn1;i++){
+		if(word_comp(terminal1[i].word,terminal2[i].word)==0){
+			Error("Words unmatch (%s|%s)\n",terminal1[i].word,
+				  terminal2[i].word);
+			individual_result(0,0,0,0,0,0);
+			return;
+		}
+    }
+	
+    /* massage the data */
+    /*------------------*/
+    massage_data();
+	
+    /* matching brackets */
+    /*-------------------*/
+    match = 0;
+    for(i=0;i<bn1;i++){
+		for(j=0;j<bn2;j++){
+			
+			if (DEBUG>1)
+				printf("1.res=%d, 2.res=%d, 1.start=%d, 2.start=%d, 1.end=%d, 2.end=%d\n",bracket1[i].result,bracket2[j].result,bracket1[i].start,bracket2[j].start,bracket1[i].end,bracket2[j].end);
+			
+			// does bracket match?
+			if(bracket1[i].result != 5 && 
+			   bracket2[j].result == 0 &&
+			   bracket1[i].start == bracket2[j].start && bracket1[i].end == bracket2[j].end) {
+				
+				// (1) do we not care about the label or (2) does the label match?
+				if (F_label==0 || label_comp(bracket1[i].label,bracket2[j].label)==1) {
+					bracket1[i].result = bracket2[j].result = 1;
+					match++;
+					match_found = 1;
+					break;
+				} else {
+					if (DEBUG>1) {
+						printf("  LABEL[%d-%d]: ",bracket1[i].start,bracket1[i].end-1);
+						l = bracket1[i].buf_end-bracket1[i].buf_start;
+						strncpy(my_buf,buf1+bracket1[i].buf_start,l);
+						my_buf[l] = '\0';
+						printf("%s\n",my_buf);
+					}
+					match_found = 1;
+					match_j[j] = 1;
+				}
+			}
+		}
+		
+		if (!match_found && bracket1[i].result != 5 && DEBUG>1) {
+			/* ### ML 09/28/03: gold bracket with no corresponding test bracket */
+			printf("  BRACKET[%d-%d]: ",bracket1[i].start,bracket1[i].end-1);
+			l = bracket1[i].buf_end-bracket1[i].buf_start;
+			strncpy(my_buf,buf1+bracket1[i].buf_start,l);
+			my_buf[l] = '\0';
+			printf("%s\n",my_buf);
+		}
+		match_found = 0;
+    }
+	
+    for(j=0;j<bn2;j++){
+		if (bracket2[j].result==0 && !match_j[j] && DEBUG>1) {
+			/* test bracket with no corresponding gold bracket */
+			printf("  EXTRA[%d-%d]: ",bracket2[j].start,bracket2[j].end-1);
+			l = bracket2[j].buf_end-bracket2[j].buf_start;
+			strncpy(my_buf,buf+bracket2[j].buf_start,l);
+			my_buf[l] = '\0';
+			printf("%s\n",my_buf);
+		}
+    }
+	
+    /* crossing */
+    /*----------*/
+    crossing = 0;
+	
+    /* crossing is counted based on the brackets */
+    /* in test rather than gold file (by Mike)   */
+    for(j=0;j<bn2;j++){
+		for(i=0;i<bn1;i++){
+			if(bracket1[i].result != 5 &&
+			   bracket2[j].result != 5 &&
+			   ((bracket1[i].start < bracket2[j].start &&
+				 bracket1[i].end   > bracket2[j].start &&
+				 bracket1[i].end   < bracket2[j].end) ||
+				(bracket1[i].start > bracket2[j].start &&
+				 bracket1[i].start < bracket2[j].end &&
+				 bracket1[i].end   > bracket2[j].end))){
+					
+					/* ### ML 09/01/03: get details on cross-brackettings */
+					if (i != last_i) {
+						if (DEBUG>1) {
+							printf("  CROSSING[%d-%d]: ",bracket1[i].start,bracket1[i].end-1);
+							l = bracket1[i].buf_end-bracket1[i].buf_start;
+							strncpy(my_buf,buf1+bracket1[i].buf_start,l);
+							my_buf[l] = '\0';
+							printf("%s\n",my_buf);
+							
+							/* ML
+							 printf("\n  CROSSING at bracket %d:\n",i-1);
+							 printf("  GOLD (tokens %d-%d): ",bracket1[i].start,bracket1[i].end-1);
+							 l = bracket1[i].buf_end-bracket1[i].buf_start;
+							 strncpy(my_buf,buf1+bracket1[i].buf_start,l);
+							 my_buf[l] = '\0';
+							 printf("%s\n",my_buf);
+							 */
+						}
+						last_i = i;
+					}
+					
+					/* ML
+					 printf("  TEST (tokens %d-%d): ",bracket2[j].start,bracket2[j].end-1);
+					 l = bracket2[j].buf_end-bracket2[j].buf_start;
+					 strncpy(my_buf,buf+bracket2[j].buf_start,l);
+					 my_buf[l] = '\0';
+					 printf("%s\n",my_buf);
+					 */
+					
+					crossing++;
+					break;
+				}
+		}
+    }
+	
+    /* Tagging accuracy */
+    /*------------------*/
+    correct_tag=0;
+    for(i=0;i<wn1;i++){
+		if(label_comp(terminal1[i].label,terminal2[i].label)==1){
+			terminal1[i].result = terminal2[i].result = 1;
+			correct_tag++;
+		} else {
+			terminal1[i].result = terminal2[i].result = 0;
+		}
+    }
+	
+    individual_result(wn1,r_bn1,r_bn2,match,crossing,correct_tag);
+}
+
+void
+fix_quote()
+{
+	int i,j,k;
+	if (DEBUG>1) {
+		for(i=0;i<MAX_QUOTE_TERM;i++){
+			if (quotterm1[i].index!=-1)
+				printf("%d: %s - %s\n",quotterm1[i].index,
+					   quotterm1[i].term.label,
+					   quotterm1[i].term.word);
+			if (quotterm2[i].index!=-1)
+				printf("%d: %s - %s\n",quotterm2[i].index,
+					   quotterm2[i].term.label,
+					   quotterm2[i].term.word);
+		}
+	}
+	for(i=0;i<MAX_QUOTE_TERM;i++) {
+		int ind = quotterm2[i].index;
+		if (ind!=-1) {
+			for(j=0;j<MAX_QUOTE_TERM;j++){
+				if (quotterm1[j].index==ind &&
+					strcmp(quotterm1[j].term.label,
+						   quotterm2[i].term.label)!=0) {
+						if (is_deletelabel(quotterm1[j].term.label) && !is_deletelabel(quotterm2[i].term.label)) {
+							reinsert_term(&quotterm1[j],terminal1,bracket1,&wn1);
+							for (k=j;k<MAX_QUOTE_TERM;k++)
+								if (quotterm1[k].index!=-1)
+									quotterm1[k].index++;
+						} else if (is_deletelabel(quotterm2[i].term.label) && !is_deletelabel(quotterm1[j].term.label)) {
+							reinsert_term(&quotterm2[i],terminal2,bracket2,&wn2);
+							for (k=i;k<MAX_QUOTE_TERM;k++)
+								if (quotterm2[k].index!=-1)
+									quotterm2[k].index++;
+						}
+					}
+			}
+		} else break;
+	}
+}
+
+void
+reinsert_term(quot,terminal,bracket,wn)
+s_term_ind* quot;
+s_terminal terminal[];
+s_bracket bracket[];
+int* wn;
+{
+	int ind = quot->index;
+	int bra = quot->bracket;
+	s_terminal* term = &quot->term;
+	int k;
+	memmove(&terminal[ind+1],
+			&terminal[ind],
+			sizeof(s_terminal)*(MAX_WORD_IN_SENT-ind-1));
+	strcpy(terminal[ind].label,term->label);
+	strcpy(terminal[ind].word,term->word);
+	(*wn)++;
+	if (DEBUG>1)
+		printf("bra=%d, ind=%d\n",bra,ind);
+	for(k=0;k<MAX_BRACKET_IN_SENT;k++) {
+		if (bracket[k].start==-1)
+			break;
+		if (DEBUG>1)
+			printf("bracket[%d]={%d,%d}\n",k,bracket[k].start,bracket[k].end);
+		if (k>=bra) {
+			bracket[k].start++;
+			bracket[k].end++;
+		}
+		//if (bracket[k].start<=ind && bracket[k].end>=ind)
+		//bracket[k].end++;
+	}
+	if (DEBUG>1)
+		printf("endslen=%d\n",quot->endslen);
+	for(k=0;k<quot->endslen;k++) {
+		//printf("ends[%d]=%d",k,quot->ends[k]);
+		bracket[quot->ends[k]].end++;
+	}
+	//free(quot->ends);
+}
+/*
+ void
+ adjust_end(ind,bra)
+ int ind;
+ int bra;
+ {
+ for(k=0;k<MAX_BRACKET_IN_SENT;k++) {
+ if (bracket[k].start==-1)
+ break;
+ printf("bracket[%d]={%d,%d}\n",k,bracket[k].start,bracket[k].end);
+ if (k>=bra)
+ bracket[k].end++;
+ }
+ }
+ */
+
+
+
+int massage_data_gold_only(){
+	int i, j;
+	int gold_valid_bracket=0;
+	char buflabel[MAX_LABEL_LEN]; // djame
+    /* for GOLD */
+    /*----------*/ 
+    for(i=0;i<bn1;i++){
+		
+		bracket1[i].result = 0;
+		
+		/* Zero element */
+		if(bracket1[i].start == bracket1[i].end){
+			//bracket1[i].result = bracket1[i].result; // was 5
+			continue;
+		}else {
+			gold_valid_bracket++;
+		}
+
+		
+        /* Modify label */
+		strcpy(buflabel,bracket1[i].label); //djame
+		modify_label(buflabel); // Djamé will be called twice
+		
+		/* Delete label */
+		for(j=0;j<Delete_label_n;j++){
+			if(label_comp(buflabel,Delete_label[j])!=1){
+				gold_valid_bracket++;
+			}
+		}
+    }	
+	
+	return gold_valid_bracket;
+}
+
+
+
+
+
+void
+massage_data()
+{
+    int i, j;
+	
+    /* for GOLD */
+    /*----------*/ 
+    for(i=0;i<bn1;i++){
+		
+		bracket1[i].result = 0;
+		
+		/* Zero element */
+		if(bracket1[i].start == bracket1[i].end){
+			bracket1[i].result = 5;
+			continue;
+		}
+		
+        /* Modify label */
+		modify_label(bracket1[i].label);
+		
+		/* Delete label */
+		for(j=0;j<Delete_label_n;j++){
+			if(label_comp(bracket1[i].label,Delete_label[j])==1){
+				bracket1[i].result = 5;
+			}
+		}
+    }
+	
+    /* for TEST */
+    /*----------*/ 
+    for(i=0;i<bn2;i++){
+		
+		bracket2[i].result = 0;
+		
+		/* Zero element */
+		if(bracket2[i].start == bracket2[i].end){
+			bracket2[i].result = 5;
+			continue;
+		}
+		
+        /* Modify label */
+		modify_label(bracket2[i].label);
+		
+		/* Delete label */
+		for(j=0;j<Delete_label_n;j++){
+			if(label_comp(bracket2[i].label,Delete_label[j])==1){
+				bracket2[i].result = 5;
+			}
+		}
+    }
+	
+	
+    /* count up real number of brackets (exclude deleted ones) */
+    /*---------------------------------------------------------*/
+    r_bn1 = r_bn2 = 0;
+	
+    for(i=0;i<bn1;i++){
+		if(bracket1[i].result != 5){
+			r_bn1++;
+		}
+    }
+	
+    for(i=0;i<bn2;i++){
+		if(bracket2[i].result != 5){
+			r_bn2++;
+		}
+    }
+}
+
+
+/*------------------------*/
+/* trim the tail of label */
+/*------------------------*/
+void
+modify_label(label)
+char *label;
+{
+    char *p;
+	
+    for(p=label;*p!='\0';p++){
+		if(*p=='-' || *p=='='|| *p=='#'){ // for dealing with  morph features
+			*p='\0';
+			break;
+		}
+    }
+}
+
+
+/*-----------------------------------------------*/
+/* add individual statistics to TOTAL statictics */
+/*-----------------------------------------------*/
+void
+individual_result(wn1,bn1,bn2,match,crossing,correct_tag)
+int wn1,bn1,bn2,match,crossing,correct_tag;
+{
+	
+    /* Statistics for ALL */
+    /*--------------------*/
+    TOTAL_sent++;
+    if(Status==1){
+		TOTAL_error_sent++;
+    }else if(Status==2){
+		TOTAL_skip_sent++;
+    }else{
+		TOTAL_bn1 += bn1;
+		TOTAL_bn2 += bn2;
+		TOTAL_match += match;
+		if(bn1==bn2 && bn2==match){
+			TOTAL_comp_sent++;
+		}
+		TOTAL_word += wn1;
+		TOTAL_crossing += crossing;
+		if(crossing==0){
+			TOTAL_no_crossing++;
+		}
+		if(crossing <= 2){
+			TOTAL_2L_crossing++;
+		}
+		TOTAL_correct_tag += correct_tag;
+    }
+	
+	
+    /* Statistics for sent length <= TOT_cut_len */
+    /*-------------------------------------------*/
+	//fprintf(stderr,"cut-off %d\n",TOT_cut_len);
+	//exit(0);
+    if(r_wn1<=TOT_cut_len){
+		TOT40_sent++;
+		if(Status==1){
+			TOT40_error_sent++;
+		}else if(Status==2){
+			TOT40_skip_sent++;
+		}else{
+			TOT40_bn1 += bn1;
+			TOT40_bn2 += bn2;
+			TOT40_match += match;
+			if(bn1==bn2 && bn2==match){
+				TOT40_comp_sent++;
+			}
+			TOT40_word += wn1;
+			TOT40_crossing += crossing;
+			if(crossing==0){
+				TOT40_no_crossing++;
+			}
+			if(crossing <= 2){
+				TOT40_2L_crossing++;
+			}
+			TOT40_correct_tag += correct_tag;
+		}
+    }
+	
+    /* Print individual result */
+    /*-------------------------*/
+    printf("%4d  %3d    %d  ",Line,r_wn1,Status);
+    printf("%6.2f %6.2f   %3d    %3d  %3d    %3d",
+		   (r_bn1==0?0.0:100.0*match/r_bn1), 
+		   (r_bn2==0?0.0:100.0*match/r_bn2),
+		   match, r_bn1, r_bn2, crossing);
+	
+    printf("   %4d  %4d   %6.2f\n",wn1,correct_tag,
+		   (wn1==0?0.0:100.0*correct_tag/wn1));
+}
+
+
+/*------------------------*/
+/* print total statistics */
+/*------------------------*/
+void
+print_total()
+{
+    int sentn;
+    double r,p,f;
+    FILE *file;
+	
+	
+	
+	r = TOTAL_bn1>0 ? 100.0*TOTAL_match/TOTAL_bn1 : 0.0;
+	p = TOTAL_bn2>0 ? 100.0*TOTAL_match/TOTAL_bn2 : 0.0;
+	f = 2*p*r/(p+r);
+	
+	if (spmrl_compact_view == 0){
+			
+		printf("============================================================================\n");
+		
+		if(TOTAL_bn1>0 && TOTAL_bn2>0){
+			printf("                %6.2f %6.2f %6d %5d %5d  %5d",
+				   (TOTAL_bn1>0?100.0*TOTAL_match/TOTAL_bn1:0.0),
+				   (TOTAL_bn2>0?100.0*TOTAL_match/TOTAL_bn2:0.0),
+				   TOTAL_match, 
+				   TOTAL_bn1, 
+				   TOTAL_bn2,
+				   TOTAL_crossing);
+		}
+		
+		printf("  %5d %5d   %6.2f",
+			   TOTAL_word,
+			   TOTAL_correct_tag,
+			   (TOTAL_word>0?100.0*TOTAL_correct_tag/TOTAL_word:0.0));
+		
+		printf("\n");
+		if (spmrl_print_filename==0){
+			printf("=== Summary ===\n");
+		}else {
+			printf("=== Summary: %s\tvs\t%s ===\n",filename1,filename2);
+		}
+
+		
+		sentn = TOTAL_sent - TOTAL_error_sent - TOTAL_skip_sent;
+		
+		printf("\n-- All --\n");
+		printf("Number of sentence        = %6d\n",TOTAL_sent);
+		printf("Number of Error sentence  = %6d\n",TOTAL_error_sent);
+		printf("Number of Skip  sentence  = %6d\n",TOTAL_skip_sent);
+		printf("Number of Valid sentence  = %6d\n",sentn);
+		
+		//r = TOTAL_bn1>0 ? 100.0*TOTAL_match/TOTAL_bn1 : 0.0;
+		printf("Bracketing Recall         = %6.2f\n",r);
+		
+	//	p = TOTAL_bn2>0 ? 100.0*TOTAL_match/TOTAL_bn2 : 0.0;
+		printf("Bracketing Precision      = %6.2f\n",p);
+		
+	//	f = 2*p*r/(p+r);
+		printf("Bracketing FMeasure       = %6.2f\n",f);
+		
+		printf("Complete match            = %6.2f\n",
+			   (sentn>0?100.0*TOTAL_comp_sent/sentn:0.0));
+		printf("Average crossing          = %6.2f\n",
+			   (sentn>0?1.0*TOTAL_crossing/sentn:0.0));
+		printf("No crossing               = %6.2f\n",
+			   (sentn>0?100.0*TOTAL_no_crossing/sentn:0.0));
+		printf("2 or less crossing        = %6.2f\n",
+			   (sentn>0?100.0*TOTAL_2L_crossing/sentn:0.0));
+		printf("Tagging accuracy          = %6.2f\n",
+			   (TOTAL_word>0?100.0*TOTAL_correct_tag/TOTAL_word:0.0));
+		
+		// Write stats also to a file.
+		file = fopen("status", "w");
+		fprintf(file, "---\n");
+		fprintf(file, "F1: %.2f\n", f);
+		fprintf(file, "LP: %.2f\n", p);
+		fprintf(file, "LR: %.2f\n", r);
+		fprintf(file, "POS: %.2f\n",
+				(TOTAL_word>0?100.0*TOTAL_correct_tag/TOTAL_word:0.0));
+		fprintf(file, "errorRate: %.2f\n", 100-f);
+		fclose(file);
+		
+		sentn = TOT40_sent - TOT40_error_sent - TOT40_skip_sent;
+		
+		printf("\n-- len<=%d --\n",TOT_cut_len);
+		printf("Number of sentence        = %6d\n",TOT40_sent);
+		printf("Number of Error sentence  = %6d\n",TOT40_error_sent);
+		printf("Number of Skip  sentence  = %6d\n",TOT40_skip_sent);
+		printf("Number of Valid sentence  = %6d\n",sentn);
+		
+		
+		r = TOT40_bn1>0 ? 100.0*TOT40_match/TOT40_bn1 : 0.0;
+		printf("Bracketing Recall         = %6.2f\n",r);
+		
+		p = TOT40_bn2>0 ? 100.0*TOT40_match/TOT40_bn2 : 0.0;
+		printf("Bracketing Precision      = %6.2f\n",p);
+		
+		f = 2*p*r/(p+r);
+		printf("Bracketing FMeasure       = %6.2f\n",f);
+		
+		printf("Complete match            = %6.2f\n",
+			   (sentn>0?100.0*TOT40_comp_sent/sentn:0.0));
+		printf("Average crossing          = %6.2f\n",
+			   (sentn>0?1.0*TOT40_crossing/sentn:0.0));
+		printf("No crossing               = %6.2f\n",
+			   (sentn>0?100.0*TOT40_no_crossing/sentn:0.0));
+		printf("2 or less crossing        = %6.2f\n",
+			   (sentn>0?100.0*TOT40_2L_crossing/sentn:0.0));
+		printf("Tagging accuracy          = %6.2f\n",
+			   (TOT40_word>0?100.0*TOT40_correct_tag/TOT40_word:0.0));
+	}else { // else spmrl_compact_view
+		if (spmrl_compact_view40 ==0){
+			double pos=(TOTAL_word>0?100.0*TOTAL_correct_tag/TOTAL_word:0.0);
+			sentn = TOTAL_sent - TOTAL_error_sent - TOTAL_skip_sent;
+		
+			double EX=(sentn>0?100.0*TOTAL_comp_sent/sentn:0.0);
+		
+			printf("F1: %6.2f %%\tPrec: %6.2f %%\tRec: %6.2f %%\t",f,r,p);
+			printf("POS: %6.2f %%\tEX: %6.2f %%\tUnparsed: %6d\tSent: %6d\tfile: %s\n",pos,EX,TOTAL_skip_sent+TOTAL_error_sent,TOTAL_sent,filename2);// ICI
+		}else {
+			
+			r = TOT40_bn1>0 ? 100.0*TOT40_match/TOT40_bn1 : 0.0;
+			p = TOT40_bn2>0 ? 100.0*TOT40_match/TOT40_bn2 : 0.0;
+			f = 2*p*r/(p+r);
+			double pos=(TOT40_word>0?100.0*TOT40_correct_tag/TOT40_word:0.0);
+			sentn = TOT40_sent - TOT40_error_sent - TOT40_skip_sent;
+			double EX=(sentn>0?100.0*TOT40_comp_sent/sentn:0.0);
+				
+			printf("F1: %6.2f %%\tPrec: %6.2f %%\tRec: %6.2f %%\t",f,r,p);
+			printf("POS: %6.2f %%\tEX: %6.2f %%\tUnparsed: %6d\tSent: %6d\tfile: %s\n",pos,EX,TOT40_skip_sent+TOT40_error_sent,TOT40_sent,filename2);// ICI<#statements#>
+		}
+
+	}
+
+}
+
+
+/*--------------------------------*/
+/* display individual information */
+/*--------------------------------*/
+void
+dsp_info()
+{
+	int i, n;
+	
+	printf("-<1>---(wn1=%3d, bn1=%3d)-           ",wn1,bn1);
+	printf("-<2>---(wn2=%3d, bn2=%3d)-\n",wn2,bn2);
+	
+	n = (wn1>wn2?wn1:wn2);
+	
+	for(i=0;i<n;i++){
+		if(terminal1[i].word[0]!='\0'){
+			printf("%3d : %d : %-6s  %-16s      ",i,terminal1[i].result,
+				   terminal1[i].label,terminal1[i].word);
+		}else{
+			printf("                                        ");
+		}
+		
+		if(terminal2[i].word[0]!='\0'){
+			printf("%3d : %d : %-6s  %-16s\n",i,terminal2[i].result,
+				   terminal2[i].label,terminal2[i].word);
+		}else{
+			printf("\n");
+		}
+	}
+	printf("\n");
+	
+	n = (bn1>bn2?bn1:bn2);
+	
+	for(i=0;i<n;i++){
+		if(bracket1[i].start != -1){
+			printf("%3d : %d : %3d  %3d  %-6s      ",i,bracket1[i].result,
+				   bracket1[i].start,bracket1[i].end,
+				   bracket1[i].label);
+		} else {
+			printf("                                ");
+		}
+		
+		if(bracket2[i].start != -1){
+			printf("%3d : %d : %3d  %3d  %-6s\n",i,bracket2[i].result,
+				   bracket2[i].start,bracket2[i].end,
+				   bracket2[i].label);
+		} else {
+			printf("\n");
+		}
+	}
+	printf("\n");
+	
+	printf("========\n");
+	
+}
+
+
+/*-----------------*/
+/* some predicates */
+/*-----------------*/
+
+
+// Djamé: reimplementing isspace (while digging bug in spmrl 2013 arabic gold dev line 616)
+int my_isspace(char c){
+	//	those are Posix's sapce : "\t\n\v\f\r"
+	//return (c==' ' || c=='\n');
+	return (c==' ' || c=='\t' || c=='\r' || c=='\n' || c=='\v' || c=='\f');
+}
+
+
+
+
+int
+is_terminator(c)
+char c;
+{
+    if(isspace(c) || c=='(' || c==')'){
+		return(1);
+    }else{
+		return(0);
+    }
+}
+
+int
+is_deletelabel(s)
+char *s;
+{
+    int i;
+	
+    for(i=0;i<Delete_label_n;i++){
+		if(strcmp(s,Delete_label[i])==0){
+			return(1);
+		}
+    }
+	
+    return(0);
+}
+
+int
+is_deletelabel_for_length(s)
+char *s;
+{
+    int i;
+	
+    for(i=0;i<Delete_label_for_length_n;i++){
+		if(strcmp(s,Delete_label_for_length[i])==0){
+			return(1);
+		}
+    }
+	
+    return(0);
+}
+
+int
+is_quote_term(s,w)
+char *s;
+char *w;
+{
+    int i;
+	
+    for(i=0;i<Quote_term_n;i++){
+		if(strcmp(s,Quote_term[i])==0){
+			// Djame : Arabic word contain quote
+			if (strcmp(w,"'")==0 || strcmp(w,"\"")==0 || strcmp(w,"/")==0)
+			//if (strcmp(w,"\"")==0 || strcmp(w,"/")==0)
+				return(1);
+		}
+    }
+	
+    return(0);
+}
+
+
+/*---------------*/
+/* compare words */
+/*---------------*/
+int
+word_comp(s1,s2)
+char *s1,*s2;
+{
+    int i;
+	
+    if(strcmp(s1,s2)==0){
+		return(1);
+    }
+	
+    for(i=0;i<EQ_word_n;i++){
+		if((strcmp(s1,EQ_word[i].s1)==0 &&
+			strcmp(s2,EQ_word[i].s2)==0) ||
+		   (strcmp(s1,EQ_word[i].s2)==0 &&
+			strcmp(s2,EQ_word[i].s1)==0)){
+			   return(1);
+		   }
+    }
+	
+    return(0);
+}
+
+/*----------------*/
+/* compare labels */
+/*----------------*/
+int
+label_comp(s1,s2)
+char *s1,*s2;
+{
+    int i;
+	// Added by djame for spmrl 2013 so pos tag got filtered too
+	
+	modify_label(s1); // djame
+	modify_label(s2); // djame
+    if(strcmp(s1,s2)==0){
+		return(1);
+    }
+	
+    for(i=0;i<EQ_label_n;i++){
+		if((strcmp(s1,EQ_label[i].s1)==0 &&
+			strcmp(s2,EQ_label[i].s2)==0) ||
+		   (strcmp(s1,EQ_label[i].s2)==0 &&
+			strcmp(s2,EQ_label[i].s1)==0)){
+			   return(1);
+		   }
+    }
+	
+    return(0);
+}
+
+
+/*--------*/
+/* errors */
+/*--------*/
+void
+Error(s,arg1,arg2,arg3)
+char *s, *arg1, *arg2, *arg3;
+{
+    Status = 1;
+    fprintf(stderr,"%d : ",Line);
+    fprintf(stderr,s,arg1,arg2,arg3);
+    if(Error_count++>Max_error){
+		exit(1);
+    }
+}
+
+
+/*---------------------*/
+/* fatal error to exit */
+/*---------------------*/
+void
+Fatal(s,arg1,arg2,arg3)
+char *s, *arg1, *arg2, *arg3;
+{
+    fprintf(stderr,s,arg1,arg2,arg3);
+    exit(1);
+}
+
+
+/*-------*/
+/* Usage */
+/*-------*/
+void
+Usage()
+{
+	fprintf(stderr," evalb [-dDh][-c n][-e n][-p param_file] gold-file test-file  \n");
+	fprintf(stderr,"                                                         \n");
+	fprintf(stderr,"    Evaluate bracketing in test-file against gold-file.  \n");
+	fprintf(stderr,"    Return recall, precision, F-Measure, tag accuracy.              \n");
+	fprintf(stderr,"                                                         \n");
+	fprintf(stderr,"  <option>                                               \n");
+	fprintf(stderr,"    -d             debug mode                            \n");
+	fprintf(stderr,"    -D             debug mode plus bracketing info       \n");
+	fprintf(stderr,"    -c n           cut-off length forstatistics (def.=40)\n");
+	fprintf(stderr,"    -e n           number of error to kill (default=10)  \n");
+	fprintf(stderr,"    -p param_file  parameter file                        \n");
+	fprintf(stderr,"    -K n		   Evaluate up to  n sentences                         \n");
+	fprintf(stderr,"    -X			   Count skipped sentences brackets as not parsed  \n");
+	fprintf(stderr,"    -L			   Compact view (for use in batch mode, all sentences  \n");
+	fprintf(stderr,"    -h    help                                           \n");
+}

parsing/EVALB_SPMRL/spmrl.prm

@@ -0,0 +1,91 @@
+##------------------------------------------##
+## Debug mode                               ##
+##   0: No debugging                        ##
+##   1: print data for individual sentence  ##
+##   2: print detailed bracketing info      ##
+##------------------------------------------##
+DEBUG 0
+
+##------------------------------------------##
+## MAX error                                ##
+##    Number of error to stop the process.  ##
+##    This is useful if there could be      ##
+##    tokanization error.                   ##
+##    The process will stop when this number##
+##    of errors are accumulated.            ##
+##------------------------------------------##
+MAX_ERROR 10000
+
+##------------------------------------------##
+## Cut-off length for statistics            ##
+##    At the end of evaluation, the         ##
+##    statistics for the senetnces of length##
+##    less than or equal to this number will##
+##    be shown, on top of the statistics    ##
+##    for all the sentences                 ##
+##------------------------------------------##
+CUTOFF_LEN 70
+
+##------------------------------------------##
+## unlabeled or labeled bracketing          ##
+##    0: unlabeled bracketing               ##
+##    1: labeled bracketing                 ##
+##------------------------------------------##
+LABELED 1
+
+##------------------------------------------##
+## Delete labels                            ##
+##    list of labels to be ignored.         ##
+##    If it is a pre-terminal label, delete ##
+##    the word along with the brackets.     ##
+##    If it is a non-terminal label, just   ##
+##    delete the brackets (don't delete     ##
+##    deildrens).                           ##
+##------------------------------------------##
+DELETE_LABEL TOP
+DELETE_LABEL ROOT
+DELETE_LABEL S1  
+DELETE_LABEL -NONE-
+DELETE_LABEL VROOT 
+
+#DELETE_LABEL ,
+#DELETE_LABEL :
+#DELETE_LABEL ``
+#DELETE_LABEL ''
+#DELETE_LABEL .
+#DELETE_LABEL ?
+#DELETE_LABEL !
+#DELETE_LABEL PONCT
+
+##------------------------------------------##
+## Delete labels for length calculation     ##
+##    list of labels to be ignored for      ##
+##    length calculation purpose            ##
+##------------------------------------------##
+DELETE_LABEL_FOR_LENGTH -NONE-
+
+##------------------------------------------##
+## Labels to be considered for misquote     ##
+##    (could be possesive or quote)         ##
+##------------------------------------------##
+#QUOTE_LABEL ``
+#QUOTE_LABEL ''
+#QUOTE_LABEL POS
+
+##------------------------------------------##
+## These ones are less common, but          ##
+##    are on occasion output by parsers:    ##      
+##------------------------------------------##
+#QUOTE_LABEL NN
+#QUOTE_LABEL CD
+#QUOTE_LABEL VBZ
+#QUOTE_LABEL :
+
+##------------------------------------------##
+## Equivalent labels, words                 ##
+##     the pairs are considered equivalent  ##
+##     This is non-directional.             ##
+##------------------------------------------##
+#EQ_LABEL ADVP PRT
+
+# EQ_WORD  Example example

parsing/EVALB_SPMRL/spmrl_hebrew.prm

@@ -0,0 +1,118 @@
+##------------------------------------------##
+## Debug mode                               ##
+##   0: No debugging                        ##
+##   1: print data for individual sentence  ##
+##   2: print detailed bracketing info      ##
+##------------------------------------------##
+DEBUG 0
+
+##------------------------------------------##
+## MAX error                                ##
+##    Number of error to stop the process.  ##
+##    This is useful if there could be      ##
+##    tokanization error.                   ##
+##    The process will stop when this number##
+##    of errors are accumulated.            ##
+##------------------------------------------##
+MAX_ERROR 10000
+
+##------------------------------------------##
+## Cut-off length for statistics            ##
+##    At the end of evaluation, the         ##
+##    statistics for the senetnces of length##
+##    less than or equal to this number will##
+##    be shown, on top of the statistics    ##
+##    for all the sentences                 ##
+##------------------------------------------##
+CUTOFF_LEN 40
+
+##------------------------------------------##
+## unlabeled or labeled bracketing          ##
+##    0: unlabeled bracketing               ##
+##    1: labeled bracketing                 ##
+##------------------------------------------##
+LABELED 1
+
+##------------------------------------------##
+## Delete labels                            ##
+##    list of labels to be ignored.         ##
+##    If it is a pre-terminal label, delete ##
+##    the word along with the brackets.     ##
+##    If it is a non-terminal label, just   ##
+##    delete the brackets (don't delete     ##
+##    deildrens).                           ##
+##------------------------------------------##
+DELETE_LABEL TOP
+DELETE_LABEL ROOT
+DELETE_LABEL S1  
+DELETE_LABEL -NONE-
+DELETE_LABEL VROOT 
+#DELETE_LABEL SENT
+
+#DELETE_LABEL ,
+#DELETE_LABEL :
+#DELETE_LABEL ``
+#DELETE_LABEL ''
+#DELETE_LABEL .
+#DELETE_LABEL ?
+#DELETE_LABEL !
+#DELETE_LABEL PONCT
+
+##------------------------------------------##
+## Delete labels for length calculation     ##
+##    list of labels to be ignored for      ##
+##    length calculation purpose            ##
+##------------------------------------------##
+DELETE_LABEL_FOR_LENGTH -NONE-
+
+##------------------------------------------##
+## Labels to be considered for misquote     ##
+##    (could be possesive or quote)         ##
+##------------------------------------------##
+#QUOTE_LABEL ``
+#QUOTE_LABEL ''
+#QUOTE_LABEL POS
+
+##------------------------------------------##
+## These ones are less common, but          ##
+##    are on occasion output by parsers:    ##      
+##------------------------------------------##
+#QUOTE_LABEL NN
+#QUOTE_LABEL CD
+#QUOTE_LABEL VBZ
+#QUOTE_LABEL :
+
+##------------------------------------------##
+## Equivalent labels, words                 ##
+##     the pairs are considered equivalent  ##
+##     This is non-directional.             ##
+##------------------------------------------##
+#EQ_LABEL ADVP PRT
+
+# EQ_WORD  Example example
+DELETE_LABEL SYN_NN
+DELETE_LABEL SYN_NNP
+DELETE_LABEL SYN_NNT
+DELETE_LABEL SYN_PRP
+DELETE_LABEL SYN_JJ
+DELETE_LABEL SYN_JJT
+DELETE_LABEL SYN_RB
+DELETE_LABEL SYN_RBR
+DELETE_LABEL SYN_MOD
+DELETE_LABEL SYN_VB
+DELETE_LABEL SYN_AUX
+DELETE_LABEL SYN_AGR
+DELETE_LABEL SYN_IN
+DELETE_LABEL SYN_COM
+DELETE_LABEL SYN_REL
+DELETE_LABEL SYN_CC
+DELETE_LABEL SYN_QW
+DELETE_LABEL SYN_HAM
+DELETE_LABEL SYN_WDT
+DELETE_LABEL SYN_DT
+DELETE_LABEL SYN_CD
+DELETE_LABEL SYN_CDT
+DELETE_LABEL SYN_AT
+DELETE_LABEL SYN_H
+DELETE_LABEL SYN_FL
+DELETE_LABEL SYN_ZVL