|
|
|
|
|
|
|
|
|
|
|
|
|
package NLP::English;
|
|
|
|
use File::Basename;
|
|
use File::Spec;
|
|
|
|
|
|
|
|
$chinesePM = NLP::Chinese;
|
|
$ParseEntry = NLP::ParseEntry;
|
|
$util = NLP::utilities;
|
|
$utf8 = NLP::UTF8;
|
|
$logfile = "";
|
|
|
|
|
|
|
|
$currency_symbol_list = "\$|\xC2\xA5|\xE2\x82\xAC|\xE2\x82\xA4";
|
|
$english_resources_skeleton_dir = "";
|
|
%dummy_ht = ();
|
|
|
|
sub build_language_hashtables {
|
|
local($caller, $primary_entity_style_filename, $data_dir) = @_;
|
|
|
|
unless ($data_dir) {
|
|
$default_data_dir = "/nfs/nlg/users/textmap/brahms-ml/arabic/bin/modules/NLP";
|
|
$data_dir = $default_data_dir if -d $default_data_dir;
|
|
}
|
|
my $english_word_filename = "$data_dir/EnglishWordlist.txt";
|
|
my $default_entity_style_MT_filename = "$data_dir/EntityStyleMT-zh.txt";
|
|
my $entity_style_all_filename = "$data_dir/EntityStyleAll.txt";
|
|
my $EnglishNonNameCapWords_filename = "$data_dir/EnglishNonNameCapWords.txt";
|
|
$english_resources_skeleton_dir = "$data_dir/EnglishResources/skeleton";
|
|
%english_annotation_ht = ();
|
|
%annotation_english_ht = ();
|
|
%english_ht = ();
|
|
$CardinalMaxWithoutComma = 99999;
|
|
$CardinalMaxNonLex = 9999000;
|
|
|
|
$primary_entity_style_filename = $default_entity_style_MT_filename unless defined($primary_entity_style_filename);
|
|
if ($primary_entity_style_filename =~ /^(ar|zh)$/) {
|
|
$languageCode = $primary_entity_style_filename;
|
|
$primary_entity_style_filename
|
|
= File::Spec->catfile($data_dir, "EntityStyleMT-$languageCode.txt");
|
|
}
|
|
|
|
open(IN,$english_word_filename) || die "Can't open $english_word_filename";
|
|
while (<IN>) {
|
|
next unless $_ =~ /^s*[^#\s]/;
|
|
$_ =~ s/\s+$//;
|
|
$line = $_;
|
|
@lines = ($line);
|
|
if (($line =~ /::gpe:/)
|
|
&& (($annotation) = ($line =~ /^.*?::(.*)$/))
|
|
&& (($pre_annotation, $singular_english, $post_annotation) = ($annotation =~ /^(.*)::plural-of:([^:]+)(|::.*)\s*$/))) {
|
|
$derived_annotation = $singular_english . "::$pre_annotation$post_annotation";
|
|
|
|
push(@lines, $derived_annotation);
|
|
}
|
|
foreach $line (@lines) {
|
|
($english,@slots) = split("::",$line);
|
|
next unless defined($english);
|
|
$english =~ s/\s+$//;
|
|
$lc_english = $english;
|
|
$lc_english =~ tr/[A-Z]/[a-z]/;
|
|
$annotation = "::" . join("::",@slots) . "::";
|
|
$english_annotation_ht{$english} = $annotation;
|
|
$english_annotation_ht{$lc_english} = $annotation;
|
|
$english_annotation_ht{"_ALT_"}->{$english}->{$annotation} = 1;
|
|
$english_annotation_ht{"_ALT_"}->{$lc_english}->{$annotation} = 1;
|
|
$synt = "";
|
|
foreach $slot_value (@slots) {
|
|
($slot,$value) = ($slot_value =~ /\s*(\w[^:]+):\s*(\S.*)$/);
|
|
next unless defined($value);
|
|
$slot =~ s/\s+$//;
|
|
$value =~ s/\s+$//;
|
|
$synt = $value if $slot eq "synt";
|
|
if (defined($annotation_english_ht{$slot_value})) {
|
|
push(@{$annotation_english_ht{$slot_value}},$english);
|
|
} else {
|
|
my @elist = ($english);
|
|
$annotation_english_ht{$slot_value} = \@elist;
|
|
}
|
|
if ($synt && defined($slot_value) && ($slot ne "synt")) {
|
|
$annot = "synt:$synt" . "::$slot_value";
|
|
if (defined($annotation_english_ht{$annot})) {
|
|
push(@{$annotation_english_ht{$annot}},$english);
|
|
} else {
|
|
my @elist = ($english);
|
|
$annotation_english_ht{$annot} = \@elist;
|
|
}
|
|
$english_annotation_ht{"_EN_SYNT_"}->{$english}->{$synt}->{$slot} = $value;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
close(IN);
|
|
|
|
if (open(IN,$EnglishNonNameCapWords_filename)) {
|
|
while (<IN>) {
|
|
next unless $_ =~ /^s*[^#\s]/;
|
|
$_ =~ s/\s+$//;
|
|
$english_ht{(lc $_)}->{COMMON_NON_NAME_CAP} = 1;
|
|
}
|
|
close(IN);
|
|
} else {
|
|
print STDERR "Can't open $EnglishNonNameCapWords_filename\n";
|
|
}
|
|
|
|
foreach $style ("primary", "all") {
|
|
if ($style eq "primary") {
|
|
$entity_style_filename = $primary_entity_style_filename || $default_entity_style_MT_filename;
|
|
} elsif ($style eq "all") {
|
|
$entity_style_filename = $entity_style_all_filename;
|
|
} else {
|
|
next;
|
|
}
|
|
%ht = ();
|
|
open(IN,$entity_style_filename) || die("Can't open $entity_style_filename (stylefile)");
|
|
my $n_entries = 0;
|
|
while (<IN>) {
|
|
next unless $_ =~ /^s*[^#\s]/;
|
|
$_ =~ s/\s+$//;
|
|
($slot,$value_string) = ($_ =~ /^([^:]+):\s*(\S.*)$/);
|
|
next unless defined($value_string);
|
|
if (defined($ht{$slot})) {
|
|
print STDERR "Warning: ignoring duplicate entry for $slot in $entity_style_filename\n";
|
|
next;
|
|
}
|
|
@values = split("::", $value_string);
|
|
foreach $value (@values) {
|
|
$value =~ s/^\s+//g;
|
|
$value =~ s/\s+$//g;
|
|
}
|
|
my @values_copy = @values;
|
|
$ht{$slot} = \@values_copy;
|
|
$n_entries++;
|
|
}
|
|
|
|
close(IN);
|
|
if ($style eq "primary") {
|
|
%english_entity_style_ht = %ht;
|
|
} elsif ($style eq "all") {
|
|
%english_entity_style_all_ht = %ht;
|
|
}
|
|
}
|
|
|
|
if (defined($raw = $english_entity_style_ht{CardinalMaxWithoutComma})
|
|
&& (@styles = @{$raw}) && ($n = $styles[0]) && ($n =~ /^\d+$/) && ($n >= 999)) {
|
|
$CardinalMaxWithoutComma = $n;
|
|
}
|
|
if (defined($raw = $english_entity_style_ht{CardinalMaxNonLex})
|
|
&& (@styles = @{$raw}) && ($n = $styles[0]) && ($n =~ /^\d+$/) && ($n >= 999999)) {
|
|
$CardinalMaxNonLex = $n;
|
|
}
|
|
|
|
return (*english_annotation_ht,*annotation_english_ht,*english_entity_style_ht);
|
|
}
|
|
|
|
sub read_language_variations {
|
|
local($this, $filename, *ht) = @_;
|
|
|
|
my $n = 0;
|
|
my $line_number = 0;
|
|
if (open(IN, $filename)) {
|
|
while (<IN>) {
|
|
$line_number++;
|
|
$us = $util->slot_value_in_double_colon_del_list($_, "us");
|
|
$uk = $util->slot_value_in_double_colon_del_list($_, "uk");
|
|
$formal = $util->slot_value_in_double_colon_del_list($_, "formal");
|
|
$informal = $util->slot_value_in_double_colon_del_list($_, "informal");
|
|
if ($us && $uk) {
|
|
$ht{VARIATION_UK_US}->{$uk}->{$us} = 1;
|
|
$n++;
|
|
}
|
|
if ($informal && $formal) {
|
|
$ht{VARIATION_INFORMAL_FORMAL}->{$informal}->{$formal} = 1;
|
|
$n++;
|
|
}
|
|
}
|
|
close(IN);
|
|
|
|
}
|
|
}
|
|
|
|
sub entity_style_listing {
|
|
local($caller,$attr) = @_;
|
|
|
|
if (defined($l = $english_entity_style_ht{$attr})) {
|
|
@sl = @{$l};
|
|
if (($#sl == 0) && ($sl[0] eq "all")) {
|
|
if (defined($al = $english_entity_style_all_ht{$attr})) {
|
|
return @{$al};
|
|
} else {
|
|
return ();
|
|
}
|
|
} else {
|
|
return @sl;
|
|
}
|
|
} else {
|
|
return ();
|
|
}
|
|
}
|
|
|
|
sub is_abbreviation {
|
|
local($caller,$noun) = @_;
|
|
|
|
$result = defined($annotation_s = $english_annotation_ht{$noun})
|
|
&& ($annotation_s =~ /::abbreviation:true::/);
|
|
|
|
return $result;
|
|
}
|
|
|
|
sub noun_adv_sem {
|
|
local($caller,$noun) = @_;
|
|
|
|
return "" unless defined($annotation_s = $english_annotation_ht{$noun});
|
|
($adv_sem) = ($annotation_s =~ /::adv_sem:([-_a-z]+)::/);
|
|
return "" unless defined($adv_sem);
|
|
return $adv_sem;
|
|
}
|
|
|
|
sub numeral_value {
|
|
local($caller,$numeral) = @_;
|
|
|
|
return "" unless defined($annotation_s = $english_annotation_ht{$numeral});
|
|
($value) = ($annotation_s =~ /::value:(\d+)::/);
|
|
return "" unless defined($value);
|
|
return $value;
|
|
}
|
|
|
|
sub annot_slot_value {
|
|
local($caller,$lex, $slot) = @_;
|
|
|
|
return "" unless defined($annotation_s = $english_annotation_ht{$lex});
|
|
($value) = ($annotation_s =~ /::$slot:([-_a-z]+)(?:::.*|)\s*$/i);
|
|
return "" unless defined($value);
|
|
return $value;
|
|
}
|
|
|
|
sub annot_slot_values {
|
|
local($caller,$lex, $slot) = @_;
|
|
|
|
return () unless @annotations = keys %{$english_annotation_ht{"_ALT_"}->{$lex}};
|
|
@annot_slot_values = ();
|
|
foreach $annotation_s (@annotations) {
|
|
($value) = ($annotation_s =~ /::$slot:([^:]+)(?:::.*|)\s*$/i);
|
|
if (defined($value)) {
|
|
$value =~ s/\s*$//;
|
|
push(@annot_slot_values, $value);
|
|
}
|
|
}
|
|
return @annot_slot_values;
|
|
}
|
|
|
|
|
|
sub noun_number_form {
|
|
local($caller,$noun,$number) = @_;
|
|
|
|
$noun = "rupee" if $noun =~ /^Rs\.?$/;
|
|
$noun = "kilometer" if $noun =~ /^km$/;
|
|
$noun = "kilogram" if $noun =~ /^kg$/;
|
|
$noun = "meter" if $noun =~ /^m$/;
|
|
$noun = "second" if $noun =~ /^(s|secs?\.?)$/;
|
|
$noun = "minute" if $noun =~ /^(mins?\.?)$/;
|
|
$noun = "hour" if $noun =~ /^(h|hrs?\.?)$/;
|
|
$noun = "year" if $noun =~ /^(yrs?\.?)$/;
|
|
$noun = "degree" if $noun =~ /^(deg\.?)$/;
|
|
$noun = "foot" if $noun =~ /^(feet|ft\.?)$/;
|
|
$noun = "square kilometer" if $noun =~ /^sq\.? km/;
|
|
$noun =~ s/metre$/meter/;
|
|
$noun =~ s/litre$/liter/;
|
|
$noun =~ s/gramme$/gram/;
|
|
$noun =~ s/tonne$/ton/;
|
|
return $noun if $noun =~ /\$$/;
|
|
return $noun unless $number =~ /^[0-9.]+$/;
|
|
return $noun if $util->member($noun,"percent");
|
|
return $noun if $noun =~ /\b(yuan|renminbi|RMB|rand|won|yen|ringgit|birr)$/;
|
|
return $noun if $number <= 1;
|
|
|
|
return $noun if $caller->is_abbreviation($noun);
|
|
|
|
$noun =~ s/^(hundred|thousand|million|billion|trillion)\s+//;
|
|
return $noun if $noun =~ /^(dollar|kilometer|pound|ton|year)s$/i;
|
|
|
|
$original_noun = $noun;
|
|
|
|
$annot = "synt:noun::plural-of:$noun";
|
|
if (defined($annotation_english_ht{$annot})) {
|
|
@elist = @{$annotation_english_ht{$annot}};
|
|
return $elist[0] if @elist;
|
|
}
|
|
|
|
$noun = $noun . "s";
|
|
return $noun if $noun =~ /(a|e|o|u)ys$/;
|
|
$noun =~ s/ys$/ies/;
|
|
$noun =~ s/ss$/ses/;
|
|
$noun =~ s/xs$/xes/;
|
|
$noun =~ s/shs$/shes/;
|
|
$noun =~ s/chs$/ches/;
|
|
$noun =~ s/mans$/men/;
|
|
|
|
return $noun;
|
|
}
|
|
|
|
|
|
sub lex_candidates {
|
|
local($caller,$surf) = @_;
|
|
|
|
@lex_cands = ($surf);
|
|
$lex_cand = $surf;
|
|
$lex_cand =~ s/ies$/y/;
|
|
push(@lex_cands,$lex_cand) unless $util->member($lex_cand, @lex_cands);
|
|
$lex_cand = $surf;
|
|
$lex_cand =~ s/s$//;
|
|
push(@lex_cands,$lex_cand) unless $util->member($lex_cand, @lex_cands);
|
|
$lex_cand = $surf;
|
|
$lex_cand =~ s/es$//;
|
|
push(@lex_cands,$lex_cand) unless $util->member($lex_cand, @lex_cands);
|
|
$lex_cand = $surf;
|
|
$lex_cand =~ s/\.$//;
|
|
push(@lex_cands,$lex_cand) unless $util->member($lex_cand, @lex_cands);
|
|
$lex_cand = $surf;
|
|
$lex_cand =~ s/men$/man/;
|
|
push(@lex_cands,$lex_cand) unless $util->member($lex_cand, @lex_cands);
|
|
|
|
return @lex_cands;
|
|
}
|
|
|
|
|
|
sub pos_tag {
|
|
local($caller,$surf) = @_;
|
|
|
|
return CD if ($surf =~ /^-?[0-9,\.]+$/);
|
|
return NN if ($surf =~ /^($currency_symbol_list\d)/);
|
|
@lex_candidates = $caller->lex_candidates($surf);
|
|
|
|
foreach $lex_cand (@lex_candidates) {
|
|
if (defined($annotation_s = $english_annotation_ht{$lex_cand})) {
|
|
|
|
($synt) = ($annotation_s =~ /::synt:([^:]+)::/);
|
|
if (defined($synt)) {
|
|
if ($synt eq "art") {
|
|
return "DT";
|
|
} elsif ($synt eq "adj") {
|
|
($grade) = ($annotation_s =~ /::grade:([^:]+)::/);
|
|
if (defined($grade) && ($grade eq "superlative")) {
|
|
return "JJS";
|
|
} elsif (defined($grade) && ($grade eq "comparative")) {
|
|
return "JJR";
|
|
} else {
|
|
return "JJ";
|
|
}
|
|
} elsif ($synt eq "noun") {
|
|
if ($lex_cand eq $surf) {
|
|
return "NN";
|
|
} else {
|
|
return "NNS";
|
|
}
|
|
} elsif ($synt eq "name") {
|
|
return "NNP";
|
|
} elsif ($synt eq "cardinal") {
|
|
return "CD";
|
|
} elsif ($synt eq "ordinal") {
|
|
return "JJ";
|
|
} elsif ($synt eq "prep") {
|
|
return "IN";
|
|
} elsif ($synt eq "conj") {
|
|
return "CC";
|
|
} elsif ($synt eq "wh_pron") {
|
|
return "WP";
|
|
} elsif ($synt eq "adv") {
|
|
return "RB";
|
|
} elsif ($synt eq "genetive_particle") {
|
|
return "POS";
|
|
} elsif ($synt eq "ordinal_particle") {
|
|
return "NN";
|
|
} elsif ($synt eq "suffix_particle") {
|
|
return "NN";
|
|
} elsif ($synt =~ /^int(erjection)?$/) {
|
|
return "UH";
|
|
} elsif (($synt =~ /^punctuation$/)
|
|
&& $util->is_rare_punctuation_string_p($surf)) {
|
|
return "SYM";
|
|
} elsif ($synt =~ /\bverb$/) {
|
|
if ($surf =~ /^(is)$/) {
|
|
return "VBZ";
|
|
} else {
|
|
return "VB";
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
return "";
|
|
}
|
|
|
|
sub indef_art_filter {
|
|
local($caller,$surf) = @_;
|
|
|
|
|
|
|
|
|
|
($surf_article,$word) = ($surf =~ /^(an?) (\S+)\s*/);
|
|
if (defined($surf_article)
|
|
&& defined($word)
|
|
&& defined($annotation = $english_annotation_ht{$word})) {
|
|
($ann_article) = ($annotation =~ /::indef-article:([^:]+)::/);
|
|
if (defined($ann_article)) {
|
|
return ($surf_article eq $ann_article) ? $surf : "";
|
|
}
|
|
}
|
|
return "" if $surf =~ /\ban [bcdfghjklmnpqrstvwxyz]/;
|
|
return "" if $surf =~ /\ban (US)\b/;
|
|
return "" if $surf =~ /\ba [aeio]/;
|
|
return "" if $surf =~ /\ba (under)/;
|
|
return $surf;
|
|
}
|
|
|
|
sub wordlist_synt {
|
|
local($caller,$word) = @_;
|
|
|
|
return "" unless defined($annotation = $english_annotation_ht{$word});
|
|
($synt) = ($annotation =~ /::synt:([^:]+)::/);
|
|
return $synt || "";
|
|
}
|
|
|
|
sub qualifier_filter {
|
|
local($caller,$surf) = @_;
|
|
|
|
return "" if $surf =~ /\b(over|more than|approximately) (million|billion|trillion)/;
|
|
return "" if $surf =~ /\b(over) (once|twice)/;
|
|
return $surf;
|
|
}
|
|
|
|
sub quantity_filter {
|
|
local($caller,$surf) = @_;
|
|
|
|
return "" if $surf =~ /^(a|an)-/;
|
|
return $surf;
|
|
}
|
|
|
|
sub value_to_english {
|
|
local($caller,$number) = @_;
|
|
|
|
$result = "";
|
|
|
|
$annot = "value:$number";
|
|
if (defined($annotation_english_ht{$annot})) {
|
|
@elist = @{$annotation_english_ht{$annot}};
|
|
$result = $elist[0] if @elist;
|
|
}
|
|
|
|
return $result;
|
|
}
|
|
|
|
sub value_to_english_ordinal {
|
|
local($caller,$number) = @_;
|
|
|
|
$result = "";
|
|
|
|
$annot = "synt:ordinal::value:$number";
|
|
if (defined($annotation_english_ht{$annot})) {
|
|
@elist = @{$annotation_english_ht{$annot}};
|
|
$result = $elist[0] if @elist;
|
|
} else {
|
|
$annot = "value:$number";
|
|
if (defined($annotation_english_ht{$annot})) {
|
|
@elist = @{$annotation_english_ht{$annot}};
|
|
$cardinal = $elist[0] if @elist;
|
|
$result = $cardinal . "th";
|
|
$result =~ s/yth$/ieth/;
|
|
}
|
|
}
|
|
|
|
return $result;
|
|
}
|
|
|
|
sub english_with_synt_slot_value {
|
|
local($caller, $english, $synt, $slot) = @_;
|
|
|
|
return $english_annotation_ht{"_EN_SYNT_"}->{$english}->{$synt}->{$slot};
|
|
}
|
|
|
|
sub english_with_synt_slot_value_defined {
|
|
local($caller, $synt, $slot) = @_;
|
|
|
|
@englishes_with_synt_slot_value_defined = ();
|
|
foreach $english (keys %{$english_annotation_ht{"_EN_SYNT_"}}) {
|
|
push(@englishes_with_synt_slot_value_defined, $english)
|
|
if defined($english_annotation_ht{"_EN_SYNT_"}->{$english}->{$synt}->{$slot})
|
|
&& ! $util->member($english, @englishes_with_synt_slot_value_defined)
|
|
}
|
|
return @englishes_with_synt_slot_value_defined;
|
|
}
|
|
|
|
sub number_composed_surface_form {
|
|
local($caller,$number,$leave_num_section_p) = @_;
|
|
|
|
return "" unless $number =~ /^\d+$/;
|
|
$leave_num_section_p = 0 unless defined($leave_num_section_p);
|
|
$anchor = "1000000000000000000000000";
|
|
while (($number < $anchor) && ($anchor >= 1000000)) {
|
|
$anchor =~ s/000//;
|
|
}
|
|
|
|
return "" unless $anchor >= 1000000;
|
|
return "" unless $english = $caller->value_to_english($anchor);
|
|
$ending = $anchor;
|
|
$ending =~ s/^1000//;
|
|
return "" unless ($number =~ /$ending$/) || (($number * 1000) % $anchor) == 0;
|
|
$num_section = $number / $anchor;
|
|
if (($num_section =~ /^[1-9]0?$/) && ! $leave_num_section_p) {
|
|
$num_section_english = $caller->value_to_english($num_section);
|
|
$num_section = $num_section_english if $num_section_english;
|
|
}
|
|
$num_section = $caller->commify($num_section);
|
|
return "$num_section $english";
|
|
}
|
|
|
|
sub de_scientify {
|
|
local($caller,$number) = @_;
|
|
|
|
|
|
if ($number =~ /[eE][-+]/) {
|
|
($n,$exp) = ($number =~ /^(\d+)[eE]\+(\d+)$/);
|
|
if (defined($exp)) {
|
|
$result = $n;
|
|
foreach $i (0 .. $exp-1) {
|
|
$result .= "0"
|
|
}
|
|
return $result;
|
|
} else {
|
|
($n,$f,$exp) = ($number =~ /^(\d+)\.(\d+)[eE]\+(\d+)$/);
|
|
if (defined($exp) && ($exp >= length($f))) {
|
|
$result = "$n$f";
|
|
foreach $i (0 .. $exp-1-length($f)) {
|
|
$result .= "0";
|
|
}
|
|
return $result;
|
|
}
|
|
}
|
|
}
|
|
return $number;
|
|
}
|
|
|
|
sub commify {
|
|
local($caller,$number) = @_;
|
|
|
|
my $text = reverse $number;
|
|
$text =~ s/(\d\d\d)(?=\d)(?!\d*\.)/$1,/g;
|
|
return scalar reverse $text;
|
|
}
|
|
|
|
my %plural_rough_number_ht = (
|
|
10 => "tens",
|
|
12 => "dozens",
|
|
20 => "scores",
|
|
100 => "hundreds",
|
|
1000 => "thousands",
|
|
10000 => "tens of thousands",
|
|
100000 => "hundreds of thousands",
|
|
1000000 => "millions",
|
|
10000000 => "tens of millions",
|
|
100000000 => "hundreds of millions",
|
|
1000000000 => "billions",
|
|
10000000000 => "tens of billions",
|
|
100000000000 => "hundreds of billions",
|
|
1000000000000 => "trillions",
|
|
10000000000000 => "tens of trillions",
|
|
100000000000000 => "hundreds of trillions",
|
|
);
|
|
|
|
sub plural_rough_plural_number {
|
|
local($caller,$number) = @_;
|
|
|
|
return $plural_rough_number_ht{$number} || "";
|
|
}
|
|
|
|
my %roman_numeral_ht = (
|
|
"I" => 1,
|
|
"II" => 2,
|
|
"III" => 3,
|
|
"IIII" => 4,
|
|
"IV" => 4,
|
|
"V" => 5,
|
|
"VI" => 6,
|
|
"VII" => 7,
|
|
"VIII" => 8,
|
|
"VIIII" => 9,
|
|
"IX" => 9,
|
|
"X" => 10,
|
|
"XX" => 20,
|
|
"XXX" => 30,
|
|
"XXXX" => 40,
|
|
"XL" => 40,
|
|
"L" => 50,
|
|
"LX" => 60,
|
|
"LXX" => 70,
|
|
"LXXX" => 80,
|
|
"LXXXX" => 90,
|
|
"XC" => 90,
|
|
"C" => 100,
|
|
"CC" => 200,
|
|
"CCC" => 300,
|
|
"CCCC" => 400,
|
|
"CD" => 400,
|
|
"D" => 500,
|
|
"DC" => 600,
|
|
"DCC" => 700,
|
|
"DCCC" => 800,
|
|
"DCCCC" => 900,
|
|
"CM" => 900,
|
|
"M" => 1000,
|
|
"MM" => 2000,
|
|
"MMM" => 3000,
|
|
"MMM" => 3000,
|
|
);
|
|
|
|
sub roman_numeral_value {
|
|
local($caller,$s) = @_;
|
|
|
|
if (($m, $c, $x, $i) = ((uc $s) =~ /^(M{0,3})(C{1,4}|CD|DC{0,4}|CM|)(X{1,4}|XL|LX{0,4}|XC|)(I{1,4}|IV|VI{0,4}|IX|)$/)) {
|
|
$sum = ($roman_numeral_ht{$m} || 0)
|
|
+ ($roman_numeral_ht{$c} || 0)
|
|
+ ($roman_numeral_ht{$x} || 0)
|
|
+ ($roman_numeral_ht{$i} || 0);
|
|
return $sum;
|
|
} else {
|
|
return 0;
|
|
}
|
|
}
|
|
|
|
sub number_surface_forms {
|
|
local($caller,$number,$pe) = @_;
|
|
|
|
print STDERR "Warning from number_surface_forms: $number not a number\n"
|
|
if $logfile && !($number =~ /^(\d+(\.\d+)?|\.\d+)$/);
|
|
|
|
|
|
|
|
$pe = "" unless defined($pe);
|
|
|
|
@num_style_list = @{$english_entity_style_ht{"FollowSourceLanguageNumberStyle"}};
|
|
$follow_num_style = $util->member("yes", @num_style_list)
|
|
&& (! (($number =~ /^([1-9]|10)$/) &&
|
|
$util->member("except-small-numbers", @num_style_list)));
|
|
$num_style = ($pe) ? $pe->get("num_style") : "";
|
|
if ($follow_num_style) {
|
|
if ($num_style =~ /digits_plus_alpha/) {
|
|
if ($number =~ /^[1-9]\d?\d?000$/) {
|
|
$digital_portion = $number;
|
|
$digital_portion =~ s/000$//;
|
|
return ("$digital_portion thousand");
|
|
} elsif ($number =~ /^[1-9]\d?\d?000000$/) {
|
|
$digital_portion = $number;
|
|
$digital_portion =~ s/000000$//;
|
|
return ("$digital_portion million");
|
|
} elsif ($number =~ /^[1-9]\d?\d?000000000$/) {
|
|
$digital_portion = $number;
|
|
$digital_portion =~ s/000000000$//;
|
|
return ("$digital_portion billion");
|
|
}
|
|
} elsif ($num_style eq "digits") {
|
|
if ($number =~ /^\d{1,4}$/) {
|
|
return ($number);
|
|
}
|
|
}
|
|
}
|
|
|
|
$number = $caller->de_scientify($number);
|
|
|
|
$composed_form = $caller->number_composed_surface_form($number);
|
|
$composed_form2 = $caller->number_composed_surface_form($number,1);
|
|
$lex_form = $caller->value_to_english($number);
|
|
$commified_form = $caller->commify($number);
|
|
|
|
if ($lex_form) {
|
|
if ($number >= 1000000) {
|
|
@result = ("one $lex_form", "1 $lex_form", "a $lex_form", $lex_form, $commified_form);
|
|
push(@result, $commified_form) if ($number <= $CardinalMaxNonLex);
|
|
} elsif ($number >= 100) {
|
|
@result = ($commified_form, "one $lex_form", "a $lex_form", $lex_form);
|
|
} elsif ($number >= 10) {
|
|
@result = ($number, $lex_form);
|
|
} elsif ($number == 1) {
|
|
@result = ("a", "an", $lex_form);
|
|
} elsif ($number == 0) {
|
|
@result = ($number, $lex_form);
|
|
} else {
|
|
@result = ($lex_form);
|
|
}
|
|
} elsif ($composed_form) {
|
|
if ($composed_form eq $composed_form2) {
|
|
@result = ($composed_form);
|
|
} elsif (($number >= 10000000) && ($composed_form2 =~ /^[1-9]0/)) {
|
|
@result = ($composed_form2, $composed_form);
|
|
} else {
|
|
@result = ($composed_form, $composed_form2);
|
|
}
|
|
push(@result, $commified_form) if $number <= $CardinalMaxNonLex;
|
|
} else {
|
|
($ten,$one) = ($number =~ /^([2-9])([1-9])$/);
|
|
($hundred) = ($number =~ /^([1-9])00$/) unless defined($one);
|
|
($thousand) = ($number =~ /^([1-9]\d?)000$/) unless defined($one) || defined($hundred);
|
|
if (defined($one) && defined($ten)
|
|
&& ($part1 = $caller->value_to_english($ten * 10))
|
|
&& ($part2 = $caller->value_to_english($one))) {
|
|
$wordy_form = "$part1-$part2";
|
|
@result = ($commified_form, $wordy_form);
|
|
} elsif (defined($hundred)
|
|
&& ($part1 = $caller->value_to_english($hundred))) {
|
|
$wordy_form = "$part1 hundred";
|
|
@result = ($commified_form, $wordy_form);
|
|
} elsif (defined($thousand)
|
|
&& ($part1 = $caller->value_to_english($thousand))) {
|
|
$wordy_form = "$part1 thousand";
|
|
@result = ($commified_form, $wordy_form);
|
|
} elsif ($number =~ /^100000$/) {
|
|
@result = ($commified_form, "one hundred thousand", "a hundred thousand", "hundred thousand");
|
|
} elsif ($pe && ($pe->surf eq $number) && ($number =~ /^\d\d\d\d(\.\d+)?$/)) {
|
|
@result = ($number);
|
|
push(@result, $commified_form) unless $commified_form eq $number;
|
|
} elsif ($number =~ /^\d{4,5}$/) {
|
|
if ($commified_form eq $number) {
|
|
@result = ($number);
|
|
} else {
|
|
@result = ($commified_form, $number);
|
|
}
|
|
} else {
|
|
@result = ($commified_form);
|
|
}
|
|
}
|
|
push (@result, $number)
|
|
unless $util->member($number, @result) || ($number > $CardinalMaxWithoutComma);
|
|
|
|
|
|
|
|
if ($follow_num_style) {
|
|
my @filtered_result = ();
|
|
foreach $r (@result) {
|
|
push(@filtered_result, $r)
|
|
if (($num_style eq "digits") && ($r =~ /^\d+$/))
|
|
|| (($num_style eq "alpha") && ($r =~ /^[-\@ a-z]*$/i))
|
|
|| (($num_style eq "digits_plus_alpha") && ($r =~ /\d.*[a-z]/i));
|
|
}
|
|
@result = @filtered_result if @filtered_result;
|
|
}
|
|
|
|
if ($pe && $pe->childGloss("and")) {
|
|
@new_result = ();
|
|
foreach $r (@result) {
|
|
if ($r =~ /^and /) {
|
|
push(@new_result, $r);
|
|
} else {
|
|
push(@new_result, "and $r");
|
|
}
|
|
}
|
|
@result = @new_result;
|
|
}
|
|
return @result;
|
|
}
|
|
|
|
sub number_range_surface_forms {
|
|
local($caller,$pe) = @_;
|
|
|
|
$value = $pe->value;
|
|
$value_coord = $pe->get("value-coord");
|
|
unless ($value_coord) {
|
|
return $caller->number_surface_forms($value);
|
|
}
|
|
$prefix = "";
|
|
if ($conj = $pe->get("conj")) {
|
|
$connector = $conj;
|
|
} else {
|
|
$connector = ($value_coord == $value + 1) ? "or" : "to";
|
|
}
|
|
if ($pe->get("between")) {
|
|
$prefix = "between ";
|
|
$connector = "and";
|
|
}
|
|
|
|
$pe1 = $pe->child("head");
|
|
$pe2 = $pe->child("coord");
|
|
@result1 = $caller->number_surface_forms($value, $pe1);
|
|
@result2 = $caller->number_surface_forms($value_coord, $pe2);
|
|
@num_style_list = @{$english_entity_style_ht{"FollowSourceLanguageNumberStyle"}};
|
|
$follow_num_style = 1 if $util->member("yes", @num_style_list);
|
|
|
|
|
|
|
|
if ($follow_num_style && ($#result1 == 0) && ($#result2 == 0)) {
|
|
$range = $prefix . $result1[0] . " $connector " . $result2[0];
|
|
$util->log(" range1: $range", $logfile);
|
|
$gazillion = "thousand|million|billion|trillion";
|
|
($a,$gaz1,$b,$gaz2) = ($range =~ /^(.+) ($gazillion) ($connector .+) ($gazillion)$/);
|
|
if (defined($a) && defined($gaz1) && defined($b) && defined($gaz2) && ($gaz1 eq $gaz2)) {
|
|
$range = "$a $b $gaz1";
|
|
$util->log(" range2: $range", $logfile);
|
|
return ($range);
|
|
}
|
|
}
|
|
|
|
@result = ();
|
|
foreach $result1 (@result1) {
|
|
next if ($value >= 1000) && ($result1 =~ /^\d+$/);
|
|
foreach $result2 (@result2) {
|
|
next if $result1 =~ /^an?\b/;
|
|
push(@result, "$prefix$result1 $connector $result2")
|
|
if ($result1 =~ /^[a-z]+$/) && ($result2 =~ /^[a-z]+$/);
|
|
next if ($result1 =~ /^[a-z]/) || ($result2 =~ /^[a-z]/);
|
|
next if ($value_coord >= 1000) && ($result2 =~ /^\d+$/);
|
|
($digits1,$letters1) = ($result1 =~ /^(\d+(?:.\d+)?) ([a-z].*)$/);
|
|
($digits2,$letters2) = ($result2 =~ /^(\d+(?:.\d+)?) ([a-z].*)$/);
|
|
if (defined($digits1) && defined($letters1)
|
|
&& defined($digits2) && defined($letters2)
|
|
&& ($letters1 eq $letters2)) {
|
|
push(@result, "$prefix$digits1 $connector $digits2 $letters1");
|
|
} elsif (($result1 =~ /^\d{1,3}$/) && ($result2 =~ /^\d{1,3}$/) && !$prefix) {
|
|
push(@result, "$result1-$result2");
|
|
if ($connector eq "to") {
|
|
my $span = "$result1 to $result2";
|
|
push(@result, $span) unless $util->member($span, @result);
|
|
}
|
|
} else {
|
|
push(@result, "$prefix$result1 $connector $result2");
|
|
}
|
|
}
|
|
}
|
|
unless (@result) {
|
|
$result1 = (@result1) ? $result1[0] : $value;
|
|
$result2 = (@result2) ? $result2[0] : $value_coord;
|
|
@result = "$prefix$result1 $connector $result2";
|
|
}
|
|
return @result;
|
|
}
|
|
|
|
sub q_number_surface_forms {
|
|
local($caller,$pe) = @_;
|
|
|
|
$surf = $pe->surf;
|
|
return ($pe->gloss) unless $value = $pe->value;
|
|
if (($value >= 1961) && ($value <= 2030)
|
|
&&
|
|
(($pe->get("struct") eq "sequence of digits")
|
|
||
|
|
($surf =~ /^\d+$/))) {
|
|
$value = "$prefix $value" if $prefix = $pe->get("prefix");
|
|
@result = ("$value");
|
|
} else {
|
|
@result = $caller->number_surface_forms($value,$pe);
|
|
@result = $caller->qualify_entities($pe,@result);
|
|
}
|
|
return @result;
|
|
}
|
|
|
|
sub ordinal_surface_forms {
|
|
local($caller,$number,$exclude_cardinals_p,$exclude_adverbials_p, $pe) = @_;
|
|
|
|
if (defined($os = $english_entity_style_ht{"Ordinal"})) {
|
|
@ordinal_styles = @{$os};
|
|
} else {
|
|
return ();
|
|
}
|
|
$exclude_cardinals_p = 0 unless defined($exclude_cardinals_p);
|
|
@num_style_list = @{$english_entity_style_ht{"FollowSourceLanguageNumberStyle"}};
|
|
$follow_num_style = 1 if $util->member("yes", @num_style_list);
|
|
$num_style = ($pe) ? $pe->get("num_style") : "";
|
|
$alpha_ok = ! ($follow_num_style && ($num_style =~ /^digits$/));
|
|
my $c_number = $caller->commify($number);
|
|
my $lex_form = "";
|
|
$lex_form = $caller->value_to_english_ordinal($number) if $alpha_ok;
|
|
my $adverbial_form
|
|
= (($number =~ /^\d+$/) && ($number >= 1) && ($number <= 10)
|
|
&& $lex_form && $util->member("secondly", @ordinal_styles))
|
|
? $lex_form . "ly" : "";
|
|
my $num_form = $caller->numeric_ordinal_form($number);
|
|
my $c_num_form = $caller->numeric_ordinal_form($c_number);
|
|
my @result = ();
|
|
|
|
|
|
if ($lex_form && $util->member("second", @ordinal_styles)) {
|
|
if (! $util->member("2nd", @ordinal_styles)) {
|
|
@result = ($lex_form);
|
|
} elsif ($c_num_form ne $num_form) {
|
|
@result = ($c_num_form, $lex_form, $num_form);
|
|
} elsif ($number >= 10) {
|
|
@result = ($num_form, $lex_form);
|
|
} else {
|
|
@result = ($lex_form, $num_form);
|
|
}
|
|
} elsif ($util->member("2nd", @ordinal_styles)) {
|
|
if ($c_num_form ne $num_form) {
|
|
@result = ($c_num_form, $num_form);
|
|
} else {
|
|
@result = ($num_form);
|
|
}
|
|
}
|
|
unless ($number =~ /^\d+$/) {
|
|
print STDERR "Warning: $number not an integer (for ordinal)\n";
|
|
}
|
|
unless ($exclude_cardinals_p) {
|
|
$incl_num_card = $util->member("2", @ordinal_styles);
|
|
$incl_lex_card = $util->member("two", @ordinal_styles);
|
|
foreach $card ($caller->number_surface_forms($number)) {
|
|
if ($card =~ /^an?$/) {
|
|
|
|
} elsif ($card =~ /^[0-9,]+$/) {
|
|
push(@result, $card) if $incl_num_card;
|
|
} else {
|
|
push(@result, $card) if $incl_lex_card && $alpha_ok;
|
|
}
|
|
}
|
|
}
|
|
push(@result,$adverbial_form) if $adverbial_form && ! $exclude_adverbials_p;
|
|
push(@result, $num_form) unless @result;
|
|
return @result;
|
|
}
|
|
|
|
sub ordinal_surface_form {
|
|
local($caller,$number,$exclude_cardinals_p,$exclude_adverbials_p, $pe) = @_;
|
|
|
|
my @surf_forms = $caller->ordinal_surface_forms($number,$exclude_cardinals_p,$exclude_adverbials_p, $pe);
|
|
return (@surf_forms) ? $surf_forms[0] : $caller->numeric_ordinal_form($number);
|
|
}
|
|
|
|
sub fraction_surface_forms {
|
|
local($caller,$pe,$modp) = @_;
|
|
|
|
my @result = ();
|
|
$numerator = $pe->get("numerator");
|
|
$denominator = $pe->get("denominator");
|
|
|
|
@surf_nums = $caller->number_surface_forms($numerator,$pe);
|
|
@surf_nums = ("one") if $numerator == 1;
|
|
@surf_dens = $caller->ordinal_surface_forms($denominator,1,1);
|
|
@surf_dens = ("half") if $denominator == 2;
|
|
@surf_dens = ("quarter") if $denominator == 4;
|
|
@surf_dens = ("tenth") if $denominator == 10;
|
|
|
|
@fraction_patterns = @{$english_entity_style_ht{"Fraction"}};
|
|
if (@surf_nums && @surf_dens) {
|
|
$surf_num = $surf_nums[0];
|
|
$surf_den = $surf_dens[0];
|
|
$surf_num_den = "";
|
|
foreach $sd (@surf_dens) {
|
|
$surf_num_den = $sd if $sd =~ /^\d/;
|
|
}
|
|
$surf_den_w_proper_number = $caller->noun_number_form($surf_den, $numerator);
|
|
foreach $fp (@fraction_patterns) {
|
|
if ($fp eq "one tenth") {
|
|
push(@result, $surf_num . " " . $surf_den_w_proper_number) unless $modp;
|
|
} elsif ($fp eq "one-tenth") {
|
|
if ($modp) {
|
|
push(@result, $surf_num . "-" . $surf_den);
|
|
} else {
|
|
push(@result, $surf_num . "-" . $surf_den_w_proper_number);
|
|
}
|
|
} elsif ($fp eq "1/10") {
|
|
push(@result, $numerator . "/" . $denominator);
|
|
} elsif ($fp eq "1/10th") {
|
|
push(@result, $numerator . "/" . $surf_num_den) if $surf_num_den;
|
|
}
|
|
}
|
|
return @result;
|
|
} else {
|
|
return ($pe->gloss);
|
|
}
|
|
}
|
|
|
|
sub currency_surface_forms {
|
|
local($caller,$pe) = @_;
|
|
|
|
@currency_surf_forms = ();
|
|
return @currency_surf_forms unless $pe->sem =~ /monetary quantity/;
|
|
$unit = $pe->get("unit");
|
|
return ($pe->gloss) unless $quant = $pe->get("quant");
|
|
return ($pe->gloss) if $pe->childSem("head") eq "currency symbol";
|
|
$quant_pe = $pe->child("quant");
|
|
if ($unit =~ /^(US|Hongkong) dollar$/) {
|
|
@units = $caller->entity_style_listing($unit);
|
|
} elsif ($unit eq "yuan") {
|
|
@units = $caller->entity_style_listing("Chinese yuan");
|
|
@rmb_pos = @{$english_entity_style_ht{"Chinese RMB position"}};
|
|
@rmb_pos = ("before-number", "after-number") if $util->member("all",@units);
|
|
} else {
|
|
@units = ($unit);
|
|
}
|
|
if (($pe->sem =~ /range$/) && $quant_pe) {
|
|
@quants = $caller->number_range_surface_forms($quant_pe);
|
|
} else {
|
|
@quants = $caller->number_surface_forms($quant, $quant_pe);
|
|
}
|
|
@quants = ($quant) unless @quants;
|
|
|
|
foreach $q (@quants) {
|
|
foreach $u_sing (@units) {
|
|
$u = ($modp) ? $u_sing : $caller->noun_number_form($u_sing, $quant);
|
|
|
|
if ($u eq "RMB") {
|
|
if ($util->member("before-number", @rmb_pos)) {
|
|
if ($q =~ /^\d/) {
|
|
push(@currency_surf_forms, "RMB" . $q);
|
|
}
|
|
}
|
|
if ($util->member("after-number", @rmb_pos)) {
|
|
push(@currency_surf_forms, $q . " RMB");
|
|
}
|
|
} elsif ($u =~ /\$$/) {
|
|
if ($q =~ /^\d/) {
|
|
$currency_surf_form = $u . $q;
|
|
push(@currency_surf_forms, $currency_surf_form);
|
|
}
|
|
} else {
|
|
$new_form = "$q $u";
|
|
push(@currency_surf_forms, $new_form) if $caller->indef_art_filter($new_form);
|
|
}
|
|
}
|
|
}
|
|
@currency_surf_forms = $caller->qualify_entities($pe,@currency_surf_forms);
|
|
|
|
|
|
return @currency_surf_forms;
|
|
}
|
|
|
|
sub age_surface_forms {
|
|
local($caller,$pe, $modp) = @_;
|
|
|
|
$gloss = $pe->gloss;
|
|
@age_surf_forms = ();
|
|
return @age_surf_forms unless $pe->sem =~ /age quantity/;
|
|
$unit = $pe->get("unit");
|
|
return ($gloss) unless $quant = $pe->get("quant");
|
|
$temporal_quant_pe = $pe->child("head");
|
|
$synt = $pe->synt;
|
|
if ($synt =~ /parenthetical/) {
|
|
if ($pe->get("slashed")) {
|
|
@age_markers = $caller->entity_style_listing("ParentheticalAgeFormatSlashed");
|
|
@age_markers = $caller->entity_style_listing("ParentheticalAgeFormat") unless @age_markers;
|
|
} else {
|
|
@age_markers = $caller->entity_style_listing("ParentheticalAgeFormat");
|
|
}
|
|
return ($gloss) unless @age_markers;
|
|
foreach $a (@age_markers) {
|
|
$age_surf_form = $a;
|
|
$age_surf_form =~ s/8/$quant/;
|
|
push(@age_surf_forms, $age_surf_form);
|
|
}
|
|
} elsif (($quant =~ /^\d+$/) && ($temporal_quant_pe->sem eq "age unit")) {
|
|
@quants = $caller->number_surface_forms($quant);
|
|
@quants = ($quant) if $pe->childSurf("quant") =~ /^\d+$/;
|
|
foreach $quant2 (@quants) {
|
|
if ($modp) {
|
|
push(@age_surf_forms, "$quant2-year-old");
|
|
} else {
|
|
$plural_marker = ($quant >= 2) ? "s" : "";
|
|
push(@age_surf_forms, "$quant2 year$plural_marker old");
|
|
}
|
|
}
|
|
} elsif ($temporal_quant_pe && ($temporal_quant_pe->sem eq "temporal quantity")) {
|
|
@temporal_quants = $caller->quantity_surface_forms($temporal_quant_pe, $modp);
|
|
foreach $temporal_quant (@temporal_quants) {
|
|
push(@age_surf_forms, $temporal_quant . (($modp) ? "-" : " ") . "old");
|
|
}
|
|
} else {
|
|
return ($gloss);
|
|
}
|
|
|
|
@age_surf_forms = ($gloss) unless @age_surf_forms;
|
|
return @age_surf_forms;
|
|
}
|
|
|
|
sub occurrence_surface_forms {
|
|
local($caller,$pe,$modp) = @_;
|
|
|
|
@quantity_surf_forms = ();
|
|
return ($pe->gloss) unless $quant = $pe->get("quant");
|
|
$quant_coord = $pe->get("quant-coord");
|
|
$quant_pe = $pe->child("quant");
|
|
$unit = "time";
|
|
if (($pe->sem =~ /range$/) && $quant_pe) {
|
|
@quants = $caller->number_range_surface_forms($quant_pe);
|
|
} else {
|
|
@quants = $caller->number_surface_forms($quant, $quant_pe);
|
|
}
|
|
@quants = ($quant) unless @quants;
|
|
if ($modp) {
|
|
return () if $pe->get("qualifier") || $quant_coord;
|
|
return ("one-time") if $quant eq "1";
|
|
return ("two-time", "two-fold", "2-fold") if $quant eq "2";
|
|
} else {
|
|
if ($quant_coord) {
|
|
return $caller->qualify_entities($pe, ("once or twice"))
|
|
if $quant eq "1" and $quant_coord eq "2";
|
|
} else {
|
|
return $caller->qualify_entities($pe, ("once")) if $quant eq "1";
|
|
return $caller->qualify_entities($pe, ("twice", "two times", "2 times",
|
|
"2-fold", "two fold")) if $quant eq "2";
|
|
}
|
|
}
|
|
foreach $q (@quants) {
|
|
$u = ($modp) ? $unit : $caller->noun_number_form($unit, $quant);
|
|
$new_form = "$q $u";
|
|
if ($modp) {
|
|
|
|
if ($pe->get("qualifier")) {
|
|
$new_form = "";
|
|
} else {
|
|
$new_form =~ s/-/-to-/;
|
|
$new_form =~ s/ /-/g;
|
|
}
|
|
}
|
|
push(@quantity_surf_forms, $new_form) if $new_form;
|
|
push(@quantity_surf_forms, "$q-fold") if $q =~ /\d/ || ($quant <= 9);
|
|
}
|
|
@quantity_surf_forms = $caller->qualify_entities($pe,@quantity_surf_forms);
|
|
|
|
return @quantity_surf_forms;
|
|
}
|
|
|
|
sub quantity_surface_forms {
|
|
local($caller,$pe,$modp) = @_;
|
|
|
|
if ($pe->get("complex") eq "true") {
|
|
return () if $modp;
|
|
$quantity_surf_form = $pe->gloss;
|
|
return ($quantity_surf_form);
|
|
}
|
|
|
|
@quantity_surf_forms = ();
|
|
$sem = $pe->get("sem");
|
|
$scale = $pe->get("scale");
|
|
$scale_mod = $pe->get("scale_mod");
|
|
$unit = $pe->get("unit") || $scale;
|
|
$mod_gloss = $pe->get("mod");
|
|
return ($pe->gloss) unless $quant = $pe->get("quant");
|
|
$quant_coord = $pe->get("quant-coord");
|
|
$quant_comb = $quant_coord || $quant;
|
|
$quant_pe = $pe->child("quant");
|
|
if (defined($u_style = $english_entity_style_ht{"\u$unit"})) {
|
|
@units = @{$u_style};
|
|
} else {
|
|
@units = ($unit);
|
|
}
|
|
if (($pe->sem =~ /range$/) && $quant_pe) {
|
|
@quants = $caller->number_range_surface_forms($quant_pe);
|
|
} else {
|
|
@quants = $caller->number_surface_forms($quant, $quant_pe);
|
|
}
|
|
@quants = ($quant) unless @quants;
|
|
foreach $q (@quants) {
|
|
foreach $u_sing (@units) {
|
|
my $u = $u_sing;
|
|
if (($sem =~ /seismic quantity/) && $scale) {
|
|
$scale =~ s/(\w+)\s*/\u\L$1/g if $scale =~ /^(Richter|Mercalli)/i;
|
|
$u = "on the $scale_mod $scale scale";
|
|
$u =~ s/\s+/ /g;
|
|
} elsif (($u_sing =~ /\S/) && ! $modp) {
|
|
$u = $caller->noun_number_form($u_sing, $quant_comb);
|
|
}
|
|
|
|
@mods = ("");
|
|
@mods = ("consecutive", "in a row") if $mod_gloss eq "continuous";
|
|
foreach $mod (@mods) {
|
|
$pre_quant_mod = "";
|
|
$in_quant_mod = ($mod =~ /(consecutive)/) ? "$mod " : "";
|
|
$post_quant_mod = ($mod =~ /(in a row)/) ? " $mod" : "";
|
|
$new_form = "$pre_quant_mod$q $in_quant_mod$u$post_quant_mod";
|
|
if ($caller->is_abbreviation($u)) {
|
|
if (($pe->sem =~ /range/) && ($q =~ /^[-0-9,\. to]+$/)
|
|
&& $modp && !($new_form =~ / (to|or) /)) {
|
|
$new_form =~ s/-/-to-/;
|
|
$new_form =~ s/ /-/g;
|
|
} elsif ($q =~ /^[-0-9,\.]+$/) {
|
|
|
|
} else {
|
|
$new_form = "";
|
|
}
|
|
} elsif ($modp) {
|
|
|
|
if (($pe->get("qualifier")) || $mod) {
|
|
$new_form = "";
|
|
} elsif ($u =~ /(square|cubic|metric|short)/) {
|
|
|
|
} elsif (($pe->sem =~ /range/) && !($new_form =~ / (to|or) /)) {
|
|
$new_form =~ s/-/-to-/;
|
|
$new_form =~ s/ /-/g;
|
|
} else {
|
|
$new_form =~ s/ /-/g;
|
|
}
|
|
}
|
|
push(@quantity_surf_forms, $new_form)
|
|
if $new_form && $caller->quantity_filter($new_form) && $caller->indef_art_filter($new_form);
|
|
}
|
|
}
|
|
}
|
|
@quantity_surf_forms = $caller->qualify_entities($pe,@quantity_surf_forms);
|
|
|
|
|
|
return @quantity_surf_forms;
|
|
}
|
|
|
|
sub qualify_entities {
|
|
local($caller,$pe,@surf_forms) = @_;
|
|
|
|
$prefix = $pe->get("prefix");
|
|
$prefix_clause = ($prefix) ? "$prefix " : "";
|
|
if ($qualifier = $pe->get("qualifier")) {
|
|
$qualifier =~ s/-/ /g;
|
|
$qualifier_key = $qualifier;
|
|
$qualifier_key =~ s/(\w+)\s*/\u\L$1/g;
|
|
|
|
@new_list = ();
|
|
if (defined($value = $english_entity_style_ht{$qualifier_key})) {
|
|
@quals = @{$value};
|
|
|
|
foreach $q (@quals) {
|
|
foreach $surf_form (@surf_forms) {
|
|
$new_form = "$prefix_clause$q $surf_form";
|
|
push(@new_list, $new_form) if $caller->qualifier_filter($new_form);
|
|
}
|
|
}
|
|
return @new_list if @new_list;
|
|
} else {
|
|
@keys = sort keys %english_entity_style_ht;
|
|
|
|
foreach $surf_form (@surf_forms) {
|
|
if (($qualifier =~ /^(couple|few|lot|many|number|several|some)$/i)
|
|
&& (($art, $lex) = ($surf_form =~ /^(an?)\s+(\S|\S.*\S)\s*$/i))) {
|
|
$plural_form = $caller->noun_number_form($lex,2);
|
|
$new_form = "$prefix_clause$qualifier $plural_form";
|
|
} else {
|
|
$new_form = "$prefix_clause$qualifier $surf_form";
|
|
}
|
|
push(@new_list, $new_form) if $caller->qualifier_filter($new_form);
|
|
}
|
|
return @new_list if @new_list;
|
|
}
|
|
}
|
|
if ($prefix) {
|
|
@prefixed_surf_forms = ();
|
|
foreach $surf_form (@surf_forms) {
|
|
if ($surf_form =~ /^$prefix /) {
|
|
push(@prefixed_surf_forms, $surf_form);
|
|
} else {
|
|
push(@prefixed_surf_forms, "$prefix $surf_form");
|
|
}
|
|
}
|
|
return @prefixed_surf_forms;
|
|
} else {
|
|
return @surf_forms;
|
|
}
|
|
}
|
|
|
|
sub percent_surface_forms {
|
|
local($caller,$pe,$modp) = @_;
|
|
|
|
@percent_surf_forms = ();
|
|
return @percent_surf_forms unless $pe->sem eq "percentage";
|
|
$prefix = "";
|
|
$quant = $pe->gloss;
|
|
$quant =~ s/%$//;
|
|
$quant =~ s/^and //;
|
|
if ($pe->gloss =~ /^and /) {
|
|
$prefix = "and";
|
|
}
|
|
@percent_markers = $caller->entity_style_listing("Percentage");
|
|
@quants = $caller->number_surface_forms($quant);
|
|
@quants = ($quant) unless @quants;
|
|
foreach $p (@percent_markers) {
|
|
foreach $q (@quants) {
|
|
if ($p =~ /%$/) {
|
|
if ($q =~ /\d$/) {
|
|
$percent_surf_form = $q . "%";
|
|
$percent_surf_form = "$prefix $percent_surf_form" if $prefix;
|
|
push(@percent_surf_forms, $percent_surf_form);
|
|
push(@percent_surf_forms, "by $percent_surf_form") unless $modp || $percent_surf_form =~ /^and /;
|
|
}
|
|
} else {
|
|
if ((($p =~ /^\d/) && ($q =~ /^\d/))
|
|
||
|
|
(($p =~ /^[a-z]/) && ($q =~ /^[a-z]/))) {
|
|
if ($p =~ /percentage point/) {
|
|
if ($quant == 1) {
|
|
$percent_surf_form = $q . " percentage point";
|
|
} else {
|
|
$percent_surf_form = $q . " percentage points";
|
|
}
|
|
} else {
|
|
$percent_surf_form = $q . " percent";
|
|
}
|
|
$percent_surf_form = "$prefix $percent_surf_form" if $prefix;
|
|
$percent_surf_form =~ s/ /-/g if $modp;
|
|
push(@percent_surf_forms, $percent_surf_form);
|
|
push(@percent_surf_forms, "by $percent_surf_form") unless $modp || $percent_surf_form =~ /^and /;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
return @percent_surf_forms;
|
|
}
|
|
|
|
sub decade_century_surface_forms {
|
|
local($caller,$pe) = @_;
|
|
|
|
if ($pe->sem =~ /century/) {
|
|
$gloss = $pe->gloss;
|
|
return ("the $gloss", "in the $gloss", $gloss);
|
|
}
|
|
@decade_surf_forms = ();
|
|
return @decade_surf_forms unless $pe->sem =~ /year range\b.*\bdecade/;
|
|
@decade_markers = @{$english_entity_style_ht{"Decade"}};
|
|
@extend_decades = @{$english_entity_style_ht{"ExtendDecades"}};
|
|
@extended_decades = @{$english_entity_style_ht{"ExtendedDecade"}};
|
|
$extended_decade = (@extended_decades) ? $extended_decades[0] : "none";
|
|
|
|
$value = $pe->value;
|
|
$extended_value = "";
|
|
foreach $extend_decade (@extend_decades) {
|
|
if ($extend_decade =~ /$value$/) {
|
|
$extended_value = $extend_decade unless $extended_value eq $extend_decade;
|
|
last;
|
|
}
|
|
}
|
|
if ($sub = $pe->get("sub")) {
|
|
$sub_clause = "$sub ";
|
|
$sub_clause =~ s/(mid) /$1-/;
|
|
} else {
|
|
$sub_clause = "";
|
|
}
|
|
|
|
if (! $extended_value) {
|
|
@values = ($value);
|
|
} elsif ($extended_decade eq "ignore") {
|
|
@values = ($value);
|
|
} elsif ($extended_decade eq "only") {
|
|
@values = ($extended_value);
|
|
} elsif ($extended_decade eq "primary") {
|
|
@values = ($extended_value, $value);
|
|
} elsif ($extended_decade eq "secondary") {
|
|
@values = ($value, $extended_value);
|
|
} else {
|
|
@values = ($value);
|
|
}
|
|
foreach $v (@values) {
|
|
foreach $dm (@decade_markers) {
|
|
$dm_ending = $dm;
|
|
$dm_ending =~ s/^\d+//;
|
|
push (@decade_surf_forms, "the $sub_clause$v$dm_ending");
|
|
push (@decade_surf_forms, "in the $sub_clause$v$dm_ending");
|
|
push (@decade_surf_forms, "$sub_clause$v$dm_ending");
|
|
}
|
|
}
|
|
return @decade_surf_forms;
|
|
}
|
|
|
|
sub day_of_the_month_surface_forms {
|
|
local($caller,$pe) = @_;
|
|
|
|
@dom_surf_forms = ();
|
|
return @dom_surf_forms
|
|
unless ($pe->sem eq "day of the month")
|
|
&& ($day_number = $pe->get("day-number"));
|
|
@dom_markers = @{$english_entity_style_ht{"DayOfTheMonth"}};
|
|
foreach $dm (@dom_markers) {
|
|
$ord = $caller->numeric_ordinal_form($day_number);
|
|
if ($dm eq "on the 5th") {
|
|
push (@dom_surf_forms, "on the $ord");
|
|
} elsif ($dm eq "the 5th") {
|
|
push (@dom_surf_forms, "the $ord");
|
|
} elsif ($dm eq "5th") {
|
|
push (@dom_surf_forms, $ord);
|
|
}
|
|
}
|
|
return @dom_surf_forms;
|
|
}
|
|
|
|
sub score_surface_forms {
|
|
local($caller,$pe) = @_;
|
|
|
|
@score_surf_forms = ();
|
|
if (($score1 = $pe->get("score1"))
|
|
&& ($score2 = $pe->get("score2"))) {
|
|
@score_markers = @{$english_entity_style_ht{"ScoreMarker"}};
|
|
@score_markers = (":") unless @score_markers;
|
|
foreach $sm (@score_markers) {
|
|
push (@score_surf_forms, "$score1$sm$score2");
|
|
}
|
|
}
|
|
push(@score_surf_forms, $pe->gloss) unless @score_surf_forms;
|
|
return @score_surf_forms;
|
|
}
|
|
|
|
sub day_of_the_week_surface_forms {
|
|
local($caller,$pe) = @_;
|
|
|
|
@dom_surf_forms = ();
|
|
@dom_markers = @{$english_entity_style_ht{"DayOfTheWeek"}};
|
|
$gloss = $pe->get("gloss");
|
|
$weekday = $pe->get("weekday");
|
|
$weekday = $gloss if ($weekday eq "") && ($gloss =~ /^\S+$/);
|
|
$relday = $pe->get("relday");
|
|
$period = $pe->get("period");
|
|
foreach $dm (@dom_markers) {
|
|
if (($dm =~ /NOPERIOD/) && $period) {
|
|
$surf = "";
|
|
} elsif (($dm eq "Sunday") || ! $relday) {
|
|
$surf = $weekday;
|
|
$surf .= " $period" if $period;
|
|
} elsif ($dm =~ /morning/) {
|
|
if ($period) {
|
|
$surf = $dm;
|
|
$surf =~ s/tomorrow/$relday/;
|
|
$surf =~ s/morning/$period/;
|
|
$surf =~ s/Sunday/$weekday/;
|
|
} else {
|
|
$surf = "";
|
|
}
|
|
} else {
|
|
$surf = $dm;
|
|
if ($period) {
|
|
if ($relday eq "today") {
|
|
$core_surf = "this $period";
|
|
} else {
|
|
$core_surf = "$relday $period";
|
|
}
|
|
} else {
|
|
$core_surf = $relday;
|
|
}
|
|
$surf =~ s/tomorrow/$core_surf/;
|
|
$surf =~ s/Sunday/$weekday/;
|
|
}
|
|
$surf =~ s/yesterday night/last night/;
|
|
$surf =~ s/this noon, ($weekday)(,\s*)?/today, $1, at noon/;
|
|
$surf =~ s/this noon/today at noon/;
|
|
$surf =~ s/this night/tonight/;
|
|
$surf =~ s/\s*NOPERIOD\s*$//;
|
|
push (@dom_surf_forms, $surf) unless $util->member($surf, @dom_surf_forms) || ! $surf;
|
|
$on_weekday = "on $surf";
|
|
push (@dom_surf_forms, $on_weekday)
|
|
if ($surf eq $weekday) && ! $util->member($on_weekday, @dom_surf_forms);
|
|
}
|
|
return @dom_surf_forms;
|
|
}
|
|
|
|
sub date_surface_forms {
|
|
local($caller,$pe,$modp) = @_;
|
|
|
|
@date_surf_forms = ();
|
|
$sem = $pe->sem;
|
|
$synt = $pe->synt;
|
|
return @date_surf_forms unless $sem =~ /date(\+year)?/;
|
|
$day = $pe->get("day");
|
|
$weekday = $pe->get("weekday");
|
|
$month_name = $pe->get("month-name");
|
|
$month_number = $pe->get("month-number");
|
|
$year = $pe->get("year");
|
|
$era = $pe->get("era");
|
|
$era_clause = "";
|
|
$calendar_type = $pe->get("calendar");
|
|
$calendar_type_clause = "";
|
|
$calendar_type_clause = " AH" if $calendar_type eq "Islamic";
|
|
$ad_year = $year;
|
|
if ($era eq "Republic era") {
|
|
$ad_year = $year + 1911;
|
|
$era_clause = " (year $year of the $era)";
|
|
}
|
|
$rel = $pe->get("rel");
|
|
if ($sep = $pe->get("sep")) {
|
|
$date_surf_form = "$month_number$sep$day";
|
|
$date_surf_form .= "$sep$year" if $year;
|
|
$date_surf_form = "$weekday, $date_surf_form" if $weekday;
|
|
$date_surf_form = "on $date_surf_form" if $synt eq "pp";
|
|
return ($date_surf_form);
|
|
}
|
|
@date_months = @{$english_entity_style_ht{"DateMonth"}};
|
|
@date_days = @{$english_entity_style_ht{"DateDay"}};
|
|
@date_order = @{$english_entity_style_ht{"DateOrder"}};
|
|
foreach $m (@date_months) {
|
|
if ($m eq "September") {
|
|
$surf_month = $month_name;
|
|
} elsif ($m =~ /^Sep(\.)?$/) {
|
|
if ($month_name eq "May") {
|
|
$surf_month = $month_name;
|
|
} else {
|
|
$period_clause = ($m =~ /\.$/) ? "." : "";
|
|
$surf_month = substr($month_name, 0, 3) . $period_clause;
|
|
}
|
|
} elsif ($m =~ /^Sept(\.)?$/) {
|
|
if ($util->member($month_name, "February", "September")) {
|
|
$period_clause = ($m =~ /\.$/) ? "." : "";
|
|
$surf_month = substr($month_name, 0, 4) . $period_clause;
|
|
} else {
|
|
$surf_month = "";
|
|
}
|
|
} else {
|
|
$surf_month = "";
|
|
}
|
|
foreach $d (@date_days) {
|
|
if ($d =~ /^\d+$/) {
|
|
$surf_day = $day;
|
|
} elsif ($d =~ /^\d+[sthrd]+$/) {
|
|
$surf_day = $caller->numeric_ordinal_form($day);
|
|
} else {
|
|
$surf_day = "";
|
|
}
|
|
if ($surf_month && $surf_day) {
|
|
foreach $o (@date_order) {
|
|
if ($calendar_type eq "Islamic") {
|
|
$date_surf_form = "$surf_day $surf_month";
|
|
} elsif ($o eq "September 6, 1998") {
|
|
$date_surf_form = "$surf_month $surf_day";
|
|
} elsif ($o eq "6 September, 1998") {
|
|
$date_surf_form = "$surf_day $surf_month";
|
|
}
|
|
$date_surf_form = "$weekday, $date_surf_form" if $weekday;
|
|
$consider_on_p = 1;
|
|
if ($year) {
|
|
$date_surf_form .= "," unless $calendar_type eq "Islamic";
|
|
$date_surf_form .= " $ad_year$calendar_type_clause$era_clause";
|
|
} elsif ($rel) {
|
|
if ($rel eq "current") {
|
|
$date_surf_form = "this $date_surf_form";
|
|
} else {
|
|
$date_surf_form = "$rel $date_surf_form";
|
|
}
|
|
$consider_on_p = 0;
|
|
}
|
|
push(@date_surf_forms, $date_surf_form)
|
|
unless $util->member($date_surf_form, @date_surf_forms) || ($synt eq "pp");
|
|
if ($consider_on_p) {
|
|
$on_date_surf_form = "on $date_surf_form";
|
|
push(@date_surf_forms, $on_date_surf_form)
|
|
unless $modp || $util->member($on_date_surf_form, @date_surf_forms);
|
|
}
|
|
|
|
if (($synt eq "pp") && ($sem eq "date")) {
|
|
push(@date_surf_forms, $date_surf_form)
|
|
unless $util->member($date_surf_form, @date_surf_forms);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
return @date_surf_forms;
|
|
|
|
}
|
|
|
|
sub numeric_ordinal_form {
|
|
local($caller,$cardinal) = @_;
|
|
|
|
return $cardinal . "th" if $cardinal =~ /1\d$/;
|
|
return $cardinal . "st" if $cardinal =~ /1$/;
|
|
return $cardinal . "nd" if $cardinal =~ /2$/;
|
|
return $cardinal . "rd" if $cardinal =~ /3$/;
|
|
return $cardinal . "h" if $cardinal =~ /t$/;
|
|
$cardinal =~ s/y$/ie/;
|
|
return $cardinal . "th";
|
|
}
|
|
|
|
sub guard_urls_x045 {
|
|
local($caller, $s) = @_;
|
|
|
|
|
|
my $result = "";
|
|
while (($pre, $url, $post) = ($s =~ /^(.*?)((?:(?:https?|ftp):\/\/|mailto:)[#%-;=?-Z_-z~]*[-a-zA-Z0-9\/#])(.*)$/)) {
|
|
$result .= "$pre\x04$url\x05";
|
|
$s = $post;
|
|
}
|
|
$result .= $s;
|
|
|
|
|
|
$s = $result;
|
|
$result = "";
|
|
while (($pre, $email, $post) = ($s =~ /^(.*?[ ,;:()\/\[\]{}<>|"'])([a-z][-_.a-z0-9]*[a-z0-9]\@[a-z][-_.a-z0-9]*[a-z0-9]\.(?:[a-z]{2,}))([ .,;:?!()\/\[\]{}<>|"'].*)$/i)) {
|
|
$result .= "$pre\x04$email\x05";
|
|
$s = $post;
|
|
}
|
|
$result .= $s;
|
|
|
|
|
|
$s = $result;
|
|
$result = "";
|
|
while (($pre, $hashtag, $post) = ($s =~ /^(.*?[ .,;()\[\]{}'])([#@](?:[a-z]|\xC3[\x80-\x96\x98-\xB6\xB8-\xBF]|HHERE)(?:[_a-z0-9]|\xC3[\x80-\x96\x98-\xB6\xB8-\xBF]|[\xC4-\xC9\xCE-\xD3][\x80-\xBF]|\xE0[\xA4-\xA5][\x80-\xBF]|\xE0[\xB6-\xB7][\x80-\xBF])*(?:[a-z0-9]|\xC3[\x80-\x96\x98-\xB6\xB8-\xBF]|[\xC4-\xC9\xCE-\xD3][\x80-\xBF]|\xE0[\xA4-\xA5][\x80-\xBF]|\xE0[\xB6-\xB7][\x80-\xBF]))(.*)$/i)) {
|
|
$result .= "$pre\x04$hashtag\x05";
|
|
$s = $post;
|
|
}
|
|
$result .= $s;
|
|
|
|
|
|
$result =~ s/((?:\b(?:fig))(?:_DONTBREAK_)?\.?|\b(?:figures?|tables?|chromosomes?)|<xref\b[^<>]*\b(?:fig)\b[^<>]*>)\s*(\d+[a-z])\b/$1 \x04$2\x05/gi;
|
|
|
|
|
|
$s = $result;
|
|
$result = "";
|
|
while (($pre, $special, $post) = ($s =~ /^(.*?)(:-?\)|:-?\(|=\/=?|\?+\/\?+|=\[)(.*)$/)) {
|
|
$result .= "$pre\x04$special\x05";
|
|
$s = $post;
|
|
}
|
|
$result .= $s;
|
|
|
|
return $result;
|
|
}
|
|
|
|
sub guard_xml_tags_x0123 {
|
|
local($caller, $s) = @_;
|
|
|
|
my $result = "";
|
|
|
|
while (($pre, $tag, $post) = ($s =~ /^(.*?)(\@?<\/?(?:[a-z][-_:a-z0-9]*)(?:\s+[a-z][-_:a-z0-9]*="[^"]*")*\s*\/?>\@?|&(?:amp|gt|lt|quot);|\[(?:QUOTE|URL)=[^ \t\n\[\]]+\]|\[\/?(?:QUOTE|IMG|INDENT|URL)\]|<\$[-_a-z0-9]+\$>|<\!--.*?-->)(.*)$/si)) {
|
|
$result .= $pre;
|
|
if (($pre =~ /\S$/) && ($tag =~ /^\S/)) {
|
|
$result .= " \x01";
|
|
$result .= "\@" if ($tag =~ /^<[a-z]/i) && (! ($pre =~ /[,;(>]$/));
|
|
} else {
|
|
$result .= "\x01";
|
|
}
|
|
$guarded_tag = $tag;
|
|
$guarded_tag =~ s/ /\x02/g;
|
|
|
|
$result .= $guarded_tag;
|
|
if (($tag =~ /\S$/) && ($post =~ /^\S/)) {
|
|
$result .= "\@" if (($tag =~ /^<\//) || ($tag =~ /\/>$/)) && (! ($result =~ /\@$/)) && (! ($post =~ /^[,;)<]/));
|
|
$result .= "\x03 ";
|
|
} else {
|
|
$result .= "\x03";
|
|
}
|
|
$s = $post;
|
|
}
|
|
$result .= $s;
|
|
return $result;
|
|
}
|
|
|
|
sub restore_urls_x045_guarded_string {
|
|
local($caller, $s) = @_;
|
|
|
|
my $orig = $s;
|
|
while (($pre, $url, $post) = ($s =~ /^(.*?)\x04([^\x04\x05]*?)\x05(.*)$/)) {
|
|
$url =~ s/ \@([-:\/])/$1/g;
|
|
$url =~ s/([-:\/])\@ /$1/g;
|
|
$url =~ s/ //g;
|
|
$url =~ s/\x02/ /g;
|
|
$s = "$pre$url$post";
|
|
}
|
|
if ($s =~ /[\x04\x05]/) {
|
|
print STDERR "Removing unexpectedly unremoved x04/x05 marks from $s\n";
|
|
$s =~ s/[\x04\x05]//g;
|
|
}
|
|
return $s;
|
|
}
|
|
|
|
sub restore_xml_tags_x0123_guarded_string {
|
|
local($caller, $s) = @_;
|
|
|
|
my $result = "";
|
|
while (($pre, $tag, $post) = ($s =~ /^(.*?)\x01(.*?)\x03(.*)$/)) {
|
|
$result .= $pre;
|
|
$tag =~ s/ \@([-:\/])/$1/g;
|
|
$tag =~ s/([-:\/])\@ /$1/g;
|
|
$tag =~ s/ //g;
|
|
$tag =~ s/\x02/ /g;
|
|
$result .= $tag;
|
|
$s = $post;
|
|
}
|
|
$result .= $s;
|
|
return $result;
|
|
}
|
|
|
|
sub load_english_abbreviations {
|
|
local($caller, $filename, *ht, $verbose) = @_;
|
|
|
|
|
|
$verbose = 1 unless defined($verbose);
|
|
my $n = 0;
|
|
if (open(IN, $filename)) {
|
|
while (<IN>) {
|
|
next if /^\
|
|
s/\s*$//;
|
|
my @expansions;
|
|
if (@expansions = split(/\s*::\s*/, $_)) {
|
|
my $abbrev = shift @expansions;
|
|
$ht{IS_ABBREVIATION}->{$abbrev} = 1;
|
|
$ht{IS_LC_ABBREVIATION}->{(lc $abbrev)} = 1;
|
|
foreach $expansion (@expansions) {
|
|
$ht{ABBREV_EXPANSION}->{$abbrev}->{$expansion} = 1;
|
|
$ht{ABBREV_EXPANSION_OF}->{$expansion}->{$abbrev} = 1;
|
|
}
|
|
$n++;
|
|
}
|
|
}
|
|
close(IN);
|
|
print STDERR "Loaded $n entries from $filename\n" if $verbose;
|
|
} else {
|
|
print STDERR "Can't open $filename\n";
|
|
}
|
|
}
|
|
|
|
sub load_split_patterns {
|
|
local($caller, $filename, *ht) = @_;
|
|
|
|
|
|
my $n = 0;
|
|
if (open(IN, $filename)) {
|
|
while (<IN>) {
|
|
next if /^\
|
|
s/\s*$//;
|
|
if (($s) = ($_ =~ /^SPLIT-DASH-X\s+(\S.*\S|\S)\s*$/)) {
|
|
$ht{SPLIT_DASH_X}->{$s} = 1;
|
|
$ht{LC_SPLIT_DASH_X}->{(lc $s)} = 1;
|
|
$n++;
|
|
} elsif (($s) = ($_ =~ /^SPLIT-X-DASH\s+(\S.*\S|\S)\s*$/)) {
|
|
$ht{SPLIT_X_DASH}->{$s} = 1;
|
|
$ht{LC_SPLIT_X_DASH}->{(lc $s)} = 1;
|
|
$n++;
|
|
} elsif (($s) = ($_ =~ /^DO-NOT-SPLIT-DASH-X\s+(\S.*\S|\S)\s*$/)) {
|
|
$ht{DO_NOT_SPLIT_DASH_X}->{$s} = 1;
|
|
$ht{LC_DO_NOT_SPLIT_DASH_X}->{(lc $s)} = 1;
|
|
$n++;
|
|
} elsif (($s) = ($_ =~ /^DO-NOT-SPLIT-X-DASH\s+(\S.*\S|\S)\s*$/)) {
|
|
$ht{DO_NOT_SPLIT_X_DASH}->{$s} = 1;
|
|
$ht{LC_DO_NOT_SPLIT_X_DASH}->{(lc $s)} = 1;
|
|
$n++;
|
|
} elsif (($s) = ($_ =~ /^DO-NOT-SPLIT\s+(\S.*\S|\S)\s*$/)) {
|
|
$ht{DO_NOT_SPLIT}->{$s} = 1;
|
|
$ht{LC_DO_NOT_SPLIT}->{(lc $s)} = 1;
|
|
$n++;
|
|
} elsif (($s) = ($_ =~ /^SPLIT\s+(\S.*\S|\S)\s*$/)) {
|
|
$ht{SPLIT}->{$s} = 1;
|
|
$ht{LC_SPLIT}->{(lc $s)} = 1;
|
|
$n++;
|
|
}
|
|
}
|
|
close(IN);
|
|
print STDERR "Loaded $n entries from $filename\n";
|
|
} else {
|
|
print STDERR "Can't open $filename\n";
|
|
}
|
|
}
|
|
|
|
sub guard_abbreviations_with_dontbreak {
|
|
local($caller, $s, *ht) = @_;
|
|
|
|
my $orig = $s;
|
|
my $result = "";
|
|
while (($pre,$potential_abbrev,$period,$post) = ($s =~ /^(.*?)((?:[a-z]+\.-?)*(?:[a-z]|\xC3[\x80-\x96\x98-\xB6\xB8-\xBF]|[\xC4-\xC9\xCE-\xD3][\x80-\xBF]|\xE0[\xA4-\xA5][\x80-\xBF]|\xE0[\xB6-\xB7][\x80-\xBF])+)(\.)(.*)$/i)) {
|
|
if (($pre =~ /([-&\/0-9]|[-\/]\@ )$/)
|
|
&& (! ($pre =~ /\b[DR](?: \@)?-(?:\@ )?$/))) {
|
|
$result .= "$pre$potential_abbrev$period";
|
|
} else {
|
|
$result .= $pre . $potential_abbrev;
|
|
$potential_abbrev_with_period = $potential_abbrev . $period;
|
|
if ($ht{IS_ABBREVIATION}->{$potential_abbrev_with_period}) {
|
|
$result .= "_DONTBREAK_";
|
|
} elsif ($ht{IS_LC_ABBREVIATION}->{(lc $potential_abbrev_with_period)}) {
|
|
$result .= "_DONTBREAK_";
|
|
}
|
|
$result .= $period;
|
|
}
|
|
$s = $post;
|
|
}
|
|
$result .= $s;
|
|
$result =~ s/\b([Nn])o\.(\s*\d)/$1o_DONTBREAK_.$2/g;
|
|
return $result;
|
|
}
|
|
|
|
$alpha = "(?:[a-z]|\xCE[\xB1-\xBF]|\xC3[\x80-\x96\x98-\xB6\xB8-\xBF]|[\xC4-\xC9\xCE-\xD3][\x80-\xBF]|\xE0[\xA4-\xA5][\x80-\xBF]|\xE0[\xB6-\xB7][\x80-\xBF])";
|
|
$alphanum = "(?:[a-z0-9]|\xCE[\xB1-\xBF]|\xC3[\x80-\x96\x98-\xB6\xB8-\xBF]|[\xC4-\xC9\xCE-\xD3][\x80-\xBF]|\xE0[\xA4-\xA5][\x80-\xBF]|\xE0[\xB6-\xB7][\x80-\xBF])(?:[-_a-z0-9]|\xCE[\xB1-\xBF]|\xC3[\x80-\x96\x98-\xB6\xB8-\xBF]|[\xC4-\xC9\xCE-\xD3][\x80-\xBF]|\xE0[\xA4-\xA5][\x80-\xBF]|\xE0[\xB6-\xB7][\x80-\xBF])*(?:[a-z0-9]|\xCE[\xB1-\xBF]|\xC3[\x80-\x96\x98-\xB6\xB8-\xBF]|[\xC4-\xC9\xCE-\xD3][\x80-\xBF]|\xE0[\xA4-\xA5][\x80-\xBF]|\xE0[\xB6-\xB7][\x80-\xBF])|(?:[a-z0-9]|\xCE[\xB1-\xBF]|\xC3[\x80-\x96\x98-\xB6\xB8-\xBF]|[\xC4-\xC9\xCE-\xD3][\x80-\xBF]|\xE0[\xA4-\xA5][\x80-\xBF]|\xE0[\xB6-\xB7][\x80-\xBF])";
|
|
|
|
sub normalize_punctuation {
|
|
local($caller, $s) = @_;
|
|
|
|
$s =~ s/\xE2\x80[\x93\x94]/-/g;
|
|
$s =~ s/ \@([-\/])/$1/g;
|
|
$s =~ s/([-\/])\@ /$1/g;
|
|
return $s;
|
|
}
|
|
|
|
sub update_replace_characters_based_on_context {
|
|
local($caller, $s) = @_;
|
|
|
|
|
|
|
|
my $rest = $s;
|
|
$s = "";
|
|
while (($pre, $left, $repl_char, $right, $post) = ($rest =~ /^(.*?\s+)(\S*)(\xEF\xBF\xBD)(\S*)(\s.*)$/)) {
|
|
$s .= "$pre$left";
|
|
if (($left =~ /[a-z]$/i) && ($right =~ /^s(?:[-.,:;?!].*)?$/i)) {
|
|
$repl_char = "\xE2\x80\x99";
|
|
} elsif (($left =~ /n$/i) && ($right =~ /^t$/i)) {
|
|
$repl_char = "\xE2\x80\x99";
|
|
} elsif (($left =~ /[a-z]\s*[.]$/i) && ($right eq "")) {
|
|
$repl_char = "\xE2\x80\x9D";
|
|
} elsif (($left eq "") && ($right =~ /^[A-Z]/i)) {
|
|
$repl_char = "\xE2\x80\x9C";
|
|
}
|
|
$s .= "$repl_char$right";
|
|
$rest = $post;
|
|
}
|
|
$s .= $rest;
|
|
|
|
return $s;
|
|
}
|
|
|
|
sub tokenize {
|
|
local($caller, $s, *ht, $control) = @_;
|
|
|
|
my $local_verbose = 0;
|
|
print "Point A: $s\n" if $local_verbose;
|
|
$control = "" unless defined($control);
|
|
my $bio_p = ($control =~ /\bbio\b/);
|
|
|
|
$s = $utf8->repair_misconverted_windows_to_utf8_strings($s);
|
|
print "Point A2: $s\n" if $local_verbose;
|
|
$s = $utf8->delete_weird_stuff($s);
|
|
print "Point B: $s\n" if $local_verbose;
|
|
|
|
|
|
$s =~ s/( +)((?:<\/[a-z][-_a-z0-9]*>)+)(\S)/$2$1$3/ig;
|
|
$s =~ s/(\S)((?:<[a-z][^<>]*>)+)( +)/$1$3$2/ig;
|
|
print "Point C: $s\n" if $local_verbose;
|
|
|
|
$a_value = $ht{IS_ABBREVIATION}->{"Fig."} || "n/a";
|
|
$s = $caller->guard_abbreviations_with_dontbreak($s, *ht);
|
|
my $standard_abbrev_s = "Adm|al|Apr|Aug|Calif|Co|Dec|Dr|etc|e.g|Feb|Febr|Gen|Gov|i.e|Jan|Ltd|Lt|Mr|Mrs|Nov|Oct|Pfc|Pres|Prof|Sen|Sept|U.S.A|U.S|vs";
|
|
my $pre;
|
|
my $core;
|
|
my $post;
|
|
$s = " $core " if ($pre,$core,$post) = ($s =~ /^(\s*)(.*?)(\s*)$/i);
|
|
$s =~ s/\xE2\x80\x89/ /g;
|
|
$standard_abbrev_s =~ s/\./\\\./g;
|
|
$s =~ s/[\x01-\x05]//g;
|
|
$s = $caller->guard_urls_x045($s);
|
|
$s = $caller->guard_xml_tags_x0123($s);
|
|
$s = $caller->update_replace_characters_based_on_context($s);
|
|
$s =~ s/((?:[a-zA-Z_]|\xC3[\x80-\x96\x98-\xB6\xB8-\xBF]|[\xC4-\xC9\xCE-\xD3][\x80-\xBF]|\xE0[\xA4-\xA5][\x80-\xBF]|\xE0[\xB6-\xB7][\x80-\xBF])\.)([,;]) /$1 $2 /g;
|
|
$s =~ s/((?:[a-zA-Z_]|\xC3[\x80-\x96\x98-\xB6\xB8-\xBF]|[\xC4-\xC9\xCE-\xD3][\x80-\xBF]|\xE0[\xA4-\xA5][\x80-\xBF]|\xE0[\xB6-\xB7][\x80-\xBF])\.)(\x04)/$1 $2/g;
|
|
if ($bio_p) {
|
|
$s =~ s/(\S)((?:wt\/|onc\/)?(?:[-+]|\?+|\xE2\x80[\x93\x94])\/(?:[-+]|\?+|\xE2\x80[\x93\x94]))/$1 $2/g;
|
|
$s =~ s/((?:[-+]|\xE2\x80[\x93\x94])\/(?:[-+]|\xE2\x80[\x93\x94]))(\S)/$1 $2/g;
|
|
}
|
|
print "Point D: $s\n" if $local_verbose;
|
|
$s =~ s/(~+)/ $1 /g;
|
|
$s =~ s/((?:\xE2\x80\xB9|\xE2\x80\xBA|\xC2\xAB|\xC2\xBB|\xE2\x80\x9E)+)/ $1 /g;
|
|
$s =~ s/(``)([A-Za-z])/$1 $2/g;
|
|
$s =~ s/((?:<|<)?=+(?:>|>)?)/ $1 /g;
|
|
$s =~ s/(\\")/ $1 /g;
|
|
$s =~ s/([^\\])("+)/$1 $2 /g;
|
|
$s =~ s/([^\\])((?:\xE2\x80\x9C)+)/$1 $2 /g;
|
|
$s =~ s/([^\\])((?:\xE2\x80\x9D)+)/$1 $2 /g;
|
|
$s =~ s/((?:<|<)?-{2,}(?:>|>)?)/ $1 /g;
|
|
$s =~ s/((?:\xE2\x80\xA6)+)/ $1 /g;
|
|
print "Point E: $s\n" if $local_verbose;
|
|
foreach $_ ((1..2)) {
|
|
|
|
$s =~ s/([.,;])(:+)/$1 \@$2/g;
|
|
$s =~ s/(:+)([.,;])/$1 \@\@ $2/g;
|
|
|
|
|
|
}
|
|
print "Point F: $s\n" if $local_verbose;
|
|
$s =~ s/(\?)/ $1 /g;
|
|
$s =~ s/(\!)/ $1 /g;
|
|
$s =~ s/ +/ /g;
|
|
$s =~ s/(\$+|\xC2\xA3|\xE2\x82[\xA0-\xBE])/ $1 /g;
|
|
$s =~ s/(\xC2\xA9|\xE2\x84\xA2)/ $1 /g;
|
|
$s =~ s/(\xC2\xB2)([-.,;:!?()])/$1 $2/g;
|
|
$s =~ s/([^ ])( )/$1 $2/g;
|
|
$s =~ s/( )([^ ])/$1 $2/g;
|
|
$s =~ s/(&#\d+|&#x[0-9A-F]+);/$1_DONTBREAK_;/gi;
|
|
$s =~ s/([\@\.]\S*\d)([a-z][A-z])/$1_DONTBREAK_$2/g;
|
|
$s =~ s/ ($standard_abbrev_s)\./ $1_DONTBREAK_\./gi;
|
|
$s =~ s/ ($standard_abbrev_s) \. (\S)/ $1_DONTBREAK_\. $2/gi;
|
|
$s =~ s/\b((?:[A-Za-z]\.){1,3}[A-Za-z])\.\s+/$1_DONTBREAK_\. /g;
|
|
$s =~ s/([ ])([A-Z])\. ([A-Z])/$1$2_DONTBREAK_\. $3/;
|
|
$s =~ s/(\S\.*?[ ])([A-Z])_DONTBREAK_\. (After|All|And|But|Each|Every|He|How|In|It|My|She|So|That|The|Then|There|These|They|This|Those|We|What|When|Which|Who|Why|You)([', ])/$1$2\. $3$4/;
|
|
$s =~ s/\b(degrees C|[Ff]ig\.? \d+ ?[A-Z]|(?:plan|Scud) [A-Z])_DONTBREAK_\./$1\./g;
|
|
$s =~ s/([^-_a-z0-9])(art|fig|no|p)((?:_DONTBREAK_)?\.)(\d)/$1$2$3 $4/gi;
|
|
$s =~ s/([^-_A-Za-z0-9])(\d+(?:\.\d+)?)(?:_DONTBREAK_)?(thousand|million|billion|trillion|min|mol|sec|kg|km|g|m|p)\b/$1$2 $3/g;
|
|
$s =~ s/([^-_a-z0-9])((?:[1-9]|1[0-2])(?:[.:][0-5]\d)?)(?:_DONTBREAK_)?([ap]m\b|[ap]\.m(?:_DONTBREAK_)?\.)/$1$2 $3/gi;
|
|
print "Point H: $s\n" if $local_verbose;
|
|
|
|
$s =~ s/(\d)([a-z][A-z])/$1 $2/g;
|
|
$s =~ s/(\w|`|'|%|[a-zA-Z]\.|[a-zA-Z]_DONTBREAK_\.)(-|\xE2\x80\x93)(\w|`|')/$1 \@$2\@ $3/g;
|
|
$s =~ s/(\w|`|'|%|[a-zA-Z]\.|[a-zA-Z]_DONTBREAK_\.)(-|\xE2\x80\x93)(\w|`|')/$1 \@$2\@ $3/g;
|
|
$s =~ s/(\w)- /$1 \@- /g;
|
|
$s =~ s/ -(\w)/ -\@ $1/g;
|
|
$s =~ s/(\d):(\d)/$1 \@:\@ $2/g;
|
|
$s =~ s/(\d)\/(\d)/$1 \@\/\@ $2/g;
|
|
$s =~ s/($alphanum)\/([,;:!?])/$1 \@\/\@ $2/g;
|
|
$s =~ s/($alphanum)([-+]+)\/($alphanum)/$1$2 \@\/\@ $3/gi;
|
|
print "Point I: $s\n" if $local_verbose;
|
|
foreach $_ ((1..5)) {
|
|
$s =~ s/([ \/()])($alphanum) ?\/ ?($alphanum)([-+ \/().,;])/$1$2 \@\/\@ $3$4/gi;
|
|
}
|
|
$s =~ s/([a-zA-Z%\/\[\]]|\xC3[\x80-\x96\x98-\xB6\xB8-\xBF]|[\xC4-\xC9\xCE-\xD3][\x80-\xBF]|\xE0[\xA4-\xA5][\x80-\xBF]|\xE0[\xB6-\xB7][\x80-\xBF]|\x05|[a-zA-Z]_DONTBREAK_\.)([,;:!?])\s*(\S)/$1 $2 $3/g;
|
|
|
|
$s =~ s/( [(\[]?)(\*)([a-z0-9])/$1$2\@ $3/gi;
|
|
$s =~ s/([a-z0-9])(\*)([.,;:)\]]* )/$1 \@$2$3/gi;
|
|
print "Point J: $s\n" if $local_verbose;
|
|
|
|
|
|
if ($s =~ /[\xD8-\xDB]/) {
|
|
for (my $i=0; $i <= 1; $i++) {
|
|
$s =~ s/([\xD8-\xDB][\x80-\xBF])([,;:!?.\(\)\[\]\/]|\xD8\x8C|\xD8\x9B|\xD8\x9F|\xD9\xAA|\xC2\xAB|\xC2\xBB|\xE2[\x80-\x9F][\x80-\xBF])/$1 $2/gi;
|
|
$s =~ s/([,;:!?.\(\)\[\]\/]|\xD8\x8C|\xD8\x9B|\xD8\x9F|\xD9\xAA|\xC2\xAB|\xC2\xBB|\xE2[\x80-\x9F][\x80-\xBF])([\xD8-\xDB][\x80-\xBF])/$1 $2/gi;
|
|
}
|
|
}
|
|
$s =~ s/(\d|[a-zA-Z]|[\xD8-\xDB][\x80-\xBF])([-])([\xD8-\xDB][\x80-\xBF])/$1 \@$2\@ $3/g;
|
|
$s =~ s/(\d|[a-zA-Z])([\xD8-\xDB][\x80-\xBF])/$1 \@\@ $2/g;
|
|
print "Point K: $s\n" if $local_verbose;
|
|
|
|
|
|
$s =~ s/(\xC2[\xA1\xBF]|\xD5\x9D|\xD6\x89|\xD8[\x8C\x9B]|\xD8\x9F|\xD9[\xAA\xAC]|\xDB\x94|\xDC[\x80\x82])/ $1 /g;
|
|
$s =~ s/(\xE0\xA5[\xA4\xA5]|\xE0\xBC[\x84-\x86\x8D-\x8F\x91\xBC\xBD])/ $1 /g;
|
|
$s =~ s/(\xE1\x81[\x8A\x8B]|\xE1\x8D[\xA2-\xA6])/ $1 /g;
|
|
$s =~ s/(\xE1\x81[\x8A\x8B]|\xE1\x8D[\xA2-\xA6]|\xE1\x9F[\x94\x96])/ $1 /g;
|
|
$s =~ s/([^0-9])(5\xE2\x80\xB2)(-)([ACGTU])/$1 $2 \@$3\@ $4/g;
|
|
$s =~ s/([^0-9])([35]\xE2\x80\xB2)/$1 $2 /g;
|
|
$s =~ s/([^0-9])(\xE2\x80\xB2)/$1 $2 /g;
|
|
$s =~ s/(\xE2\x81\x99)/ $1 /g;
|
|
$s =~ s/(\xE3\x80[\x81\x82\x8A-\x91]|\xE3\x83\xBB|xEF\xB8\xB0|\xEF\xBC\x8C)/ $1 /g;
|
|
$s =~ s/(\xEF\xBC[\x81-\x8F\x9A\x9F])/ $1 /g;
|
|
print "Point L: $s\n" if $local_verbose;
|
|
|
|
$s =~ s/((?:\xE3\x80\x80)+)/ $1 /g;
|
|
$s =~ s/((?:\xE1\x8D\xA1)+)/ $1 /g;
|
|
|
|
|
|
$s =~ s/([\xF0-\xFE][\x80-\xBF]*)([\x00-\x7F\xC0-\xDF][\x80-\xBF]*)/$1 $2/g;
|
|
$s =~ s/([\x00-\x7F\xC0-\xDF][\x80-\xBF]*)([\xF0-\xFE][\x80-\xBF]*)/$1 $2/g;
|
|
print "Point M: $s\n" if $local_verbose;
|
|
|
|
$s =~ s/( \d+)([,;:!?] )/$1 $2/g;
|
|
$s =~ s/ ([,;()\[\]])([a-zA-Z0-9.,;])/ $1 $2/g;
|
|
$s =~ s/(\)+)([-\/])([a-zA-Z0-9])/$1 $2 $3/g;
|
|
$s =~ s/([0-9\*\[\]()]|\xE2\x80\xB2)([.,;:] )/$1 $2/g;
|
|
$s =~ s/([a-zA-Z%]|\xC3[\x80-\x96\x98-\xB6\xB8-\xBF]|[\xC4-\xC9\xCE-\xD3][\x80-\xBF]|\xE0[\xA4-\xA5][\x80-\xBF]|\xE0[\xB6-\xB7][\x80-\xBF]|\x05)([,;:.!?])([")]|''|\xE2\x80[\x99\x9D]|)(\s)/$1 $2 $3$4/g;
|
|
$s =~ s/([a-zA-Z%]|\xC3[\x80-\x96\x98-\xB6\xB8-\xBF]|[\xC4-\xC9\xCE-\xD3][\x80-\xBF]|\xE0[\xA4-\xA5][\x80-\xBF]|\xE0[\xB6-\xB7][\x80-\xBF]|\x05)([,;:.!?])([")]|''|\xE2\x80[\x99\x9D]|)\s*$/$1 $2 $3/g;
|
|
$s =~ s/([.,;:]|\xC3[\x80-\x96\x98-\xB6\xB8-\xBF]|[\xC4-\xC9\xCE-\xD3][\x80-\xBF]|\xE0[\xA4-\xA5][\x80-\xBF]|\xE0[\xB6-\xB7][\x80-\xBF]|\x05)('|\xE2\x80[\x99\x9D])/$1 $2/g;
|
|
$s =~ s/('|\xE2\x80[\x99\x9D])([.,;:]|\x04)/$1 $2/g;
|
|
$s =~ s/([(){}\[\]]|\xC2\xB1)/ $1 /g;
|
|
$s =~ s/([a-zA-Z0-9]|\xC3[\x80-\x96\x98-\xB6\xB8-\xBF]|[\xC4-\xC9\xCE-\xD3][\x80-\xBF]|\xE0[\xA4-\xA5][\x80-\xBF]|\xE0[\xB6-\xB7][\x80-\xBF]|\x05)\.\s*$/$1 ./g;
|
|
$s =~ s/([a-zA-Z]|\xC3[\x80-\x96\x98-\xB6\xB8-\xBF]|[\xC4-\xC9\xCE-\xD3][\x80-\xBF]|\xE0[\xA4-\xA5][\x80-\xBF]|\xE0[\xB6-\xB7][\x80-\xBF]|\x05)\.\s+/$1 . /g;
|
|
$s =~ s/([a-zA-Z]|\xC3[\x80-\x96\x98-\xB6\xB8-\xBF]|[\xC4-\xC9\xCE-\xD3][\x80-\xBF]|\xE0[\xA4-\xA5][\x80-\xBF]|\xE0[\xB6-\xB7][\x80-\xBF]|\x05)\.(\x04)/$1 . $2/g;
|
|
$s =~ s/([0-9]),\s+(\S)/$1 , $2/g;
|
|
$s =~ s/([a-zA-Z])(\$)/$1 $2/g;
|
|
$s =~ s/(\$|[~<=>]|\xC2\xB1|\xE2\x89[\xA4\xA5]|\xE2\xA9[\xBD\xBE])(\d)/$1 $2/g;
|
|
$s =~ s/(RMB)(\d)/$1 $2/g;
|
|
print "Point N: $s\n" if $local_verbose;
|
|
foreach $_ ((1..2)) {
|
|
$s =~ s/([ '"]|\xE2\x80\x9C)(are|could|did|do|does|had|has|have|is|should|was|were|would)(n't|n\xE2\x80\x99t)([ '"]|\xE2\x80\x9D)/$1 $2 $3 $4/gi;
|
|
$s =~ s/ (can)(not) / $1 $2 /gi;
|
|
$s =~ s/ (ca)\s*(n)('t|\xE2\x80\x99t) / $1$2 $2$3 /gi;
|
|
$s =~ s/ ([Ww])o\s*n('|\xE2\x80\x99)t / $1ill n$2t /g;
|
|
$s =~ s/ WO\s*N('|\xE2\x80\x99)T / WILL N$1T /g;
|
|
$s =~ s/ ([Ss])ha\s*n('|\xE2\x80\x99)t / $1hall n$2t /g;
|
|
$s =~ s/ SHAN('|\xE2\x80\x99)T / SHALL N$1T /g;
|
|
|
|
|
|
|
|
}
|
|
print "Point O: $s\n" if $local_verbose;
|
|
$s =~ s/(\d)%/$1 %/g;
|
|
$s =~ s/ '(d|ll|m|re|s|ve|em) / '_DONTBREAK_$1 /g;
|
|
$s =~ s/ \xE2\x80\x99t(d|ll|m|re|s|ve) / \xE2\x80\x99t_DONTBREAK_$1 /g;
|
|
$s =~ s/([^0-9a-z'.])('|\xE2\x80\x98)([0-9a-z])/$1$2 $3/gi;
|
|
$s =~ s/([0-9a-z])(\.(?:'|\xE2\x80\x99))([^0-9a-z']|\xE2\x80\x99)/$1 $2$3/gi;
|
|
$s =~ s/([0-9a-z]_?\.?)((?:'|\xE2\x80\x99)(?:d|ll|m|re|s|ve|))([^0-9a-z'])/$1 $2$3/gi;
|
|
$s =~ s/([("]|\xE2\x80\x9C|'')(\w)/$1 $2/g;
|
|
print "Point P: $s\n" if $local_verbose;
|
|
$s =~ s/(\w|[.,;:?!])([")]|''|\xE2\x80\x9D)/$1 $2/g;
|
|
$s =~ s/ ([,;()\[\]])([a-zA-Z0-9.,;])/ $1 $2/g;
|
|
$s =~ s/([a-z0-9]) ?(\()([-+_ a-z0-9\/]+)(\))/$1 $2 $3 $4 /ig;
|
|
$s =~ s/([a-z0-9]) ?(\[)([-+_ a-z0-9\/]+)(\])/$1 $2 $3 $4 /ig;
|
|
$s =~ s/([a-z0-9]) ?(\{)([-+_ a-z0-9\/]+)(\})/$1 $2 $3 $4 /ig;
|
|
$s =~ s/([%])-(\d+(?:\.\d+)? ?%)/$1 \@-\@ $2/g;
|
|
$s =~ s/( )(art|No)_DONTBREAK_(\.{2,})/$1 $2$3/gi;
|
|
$s =~ s/(_DONTBREAK_\.)(\.{1,})/$1 $2/g;
|
|
print "Point Q: $s\n" if $local_verbose;
|
|
foreach $_ ((1 .. 2)) {
|
|
$s =~ s/(\s(?:[-a-z0-9()']|\xC3[\x80-\x96\x98-\xB6\xB8-\xBF]|[\xC4-\xC9\xCE-\xD3][\x80-\xBF]|\xE0[\xA4-\xA5][\x80-\xBF]|\xE0[\xB6-\xB7][\x80-\xBF])*)(\.{2,})((?:[-a-z0-9()?!:\/']|\xC3[\x80-\x96\x98-\xB6\xB8-\xBF]|[\xC4-\xC9\xCE-\xD3][\x80-\xBF]|\xE0[\xA4-\xA5][\x80-\xBF]|\xE0[\xB6-\xB7][\x80-\xBF])*\s|(?:[-a-z0-9()'\/]|\xC3[\x80-\x96\x98-\xB6\xB8-\xBF]|[\xC4-\xC9\xCE-\xD3][\x80-\xBF]|\xE0[\xA4-\xA5][\x80-\xBF]|\xE0[\xB6-\xB7][\x80-\xBF])+\.\s)/$1 $2 $3/gi;
|
|
}
|
|
$s =~ s/0s\b/0 s/g;
|
|
$s =~ s/([0-9])(\x04)/$1 $2/g;
|
|
$s =~ s/ +/ /g;
|
|
print "Point R: $s\n" if $local_verbose;
|
|
|
|
if ($bio_p) {
|
|
foreach $_ ((1 .. 2)) {
|
|
$s =~ s/([a-z]) \@(-|\xE2\x80[\x93\x94])\@ (\d+(?:$alpha)?\d*\+?)([- \/])/$1$2$3$4/ig;
|
|
$s =~ s/([a-z]) \@(-|\xE2\x80[\x93\x94])\@ ((?:alpha|beta|kappa)\d+)([- \/])/$1$2$3$4/ig;
|
|
$s =~ s/([a-z]) \@(-|\xE2\x80[\x93\x94])\@ ((?:a|b|h|k)\d)([- \/])/$1$2$3$4/ig;
|
|
$s =~ s/([a-z0-9]) \@(-|\xE2\x80[\x93\x94])\@ ([a-z])([- \/])/$1$2$3$4/ig;
|
|
$s =~ s/([- \/])(\d*[a-z]) \@(-|\xE2\x80[\x93\x94])\@ ([a-z0-9])/$1$2$3$4/ig;
|
|
}
|
|
|
|
$s =~ s/(\?\/) +(\?)/$1$2/g;
|
|
$s =~ s/([^ ?])((?:wt\/|onc\/)?(?:[-+]|\?+|\xE2\x80[\x93\x94])\/(?:[-+]|\?+|\xE2\x80[\x93\x94]))/$1 $2/g;
|
|
$s =~ s/((?:[-+]|\xE2\x80[\x93\x94])\/(?:[-+]|\xE2\x80[\x93\x94]))(\S)/$1 $2/g;
|
|
|
|
|
|
$rest = $s;
|
|
$s = "";
|
|
while (($pre, $stem, $slashed_number_s, $post) = ($rest =~ /^(.*?[^-_a-z0-9])([a-z][-_a-z]*)(\d+(?:(?: \@)?\/(?:\@ )?(?:\d+))+)([^-+a-z0-9].*|)$/i)) {
|
|
if ((($pre =~ /\x04[^\x05]*$/) && ($post =~ /^[^\x04]*\x05/))
|
|
|| ($stem =~ /^(mid|pre|post|sub|to)$/i)) {
|
|
$s .= "$pre$stem$slashed_number_s";
|
|
} else {
|
|
$s .= $pre;
|
|
my @slashed_numbers = split(/(?: \@)?\/(?:\@ )?/, $slashed_number_s);
|
|
foreach $i ((0 .. $#slashed_numbers)) {
|
|
my $number = $slashed_numbers[$i];
|
|
$s .= "$stem$number";
|
|
$s .= " @\/@ " unless $i == $#slashed_numbers;
|
|
}
|
|
}
|
|
$rest = $post;
|
|
}
|
|
$s .= $rest;
|
|
|
|
|
|
while (($pre, $stem, $dash1, $number1, $dash2, $number2, $post) = ($s =~ /^(.*[^-_a-z0-9])([a-z][-_a-z]*)(?: \@)?(-|\xE2\x80[\x93\x94])(?:\@ )?(\d+)(?: \@)?\/(?:\@ )?(?:\@ )?(-|\xE2\x80[\x93\x94])(?:\@ )?(\d+)([^-+a-z0-9].*|)$/i)) {
|
|
$s = "$pre$stem$dash1$number1 \@\/\@ $stem$dash2$number2$post";
|
|
}
|
|
$rest = $s;
|
|
$s = "";
|
|
|
|
while (($pre, $stem, $dash, $slashed_letter_s, $post) = ($rest =~ /^(.*[^-_a-z0-9])([a-z][-_a-z0-9]*)(-|\xE2\x80[\x93\x94])([a-z](?:(?: \@)?\/(?:\@ )?(?:[a-z]))+)([^-+a-z0-9].*|)$/i)) {
|
|
if (($pre =~ /\x04[^\x05]*$/) && ($post =~ /^[^\x04]*\x05/)) {
|
|
$s .= "$pre$stem$dash1$number1$dash2$number2";
|
|
} else {
|
|
$s .= $pre;
|
|
my @slashed_letters = split(/(?: \@)?\/(?:\@ )?/, $slashed_letter_s);
|
|
foreach $i ((0 .. $#slashed_letters)) {
|
|
my $letter = $slashed_letters[$i];
|
|
$s .= "$stem$dash$letter";
|
|
$s .= " @\/@ " unless $i == $#slashed_letters;
|
|
}
|
|
}
|
|
$rest = $post;
|
|
}
|
|
$s .= $rest;
|
|
|
|
|
|
my $rest = $s;
|
|
my $new_s = "";
|
|
while (($pre, $dash, $right, $post) = ($rest =~ /^(.*?)(-|\xE2\x80[\x93\x94])([a-z]+)( .*|)$/i)) {
|
|
$new_s .= $pre;
|
|
if (($right eq "I") && ($pre =~ / [a-zA-Z][a-z]*$/)) {
|
|
|
|
$new_s .= " \@" . $dash . "\@ ";
|
|
} elsif ($ht{LC_SPLIT_DASH_X}->{($caller->normalize_punctuation(lc $right))}) {
|
|
$new_s .= " \@" . $dash . "\@ ";
|
|
} else {
|
|
$new_s .= $dash;
|
|
}
|
|
$new_s .= $right;
|
|
$rest = $post;
|
|
}
|
|
$new_s .= $rest;
|
|
$s = $new_s;
|
|
|
|
|
|
$rest = $s;
|
|
$new_s = "";
|
|
while (($pre, $left, $dash, $post) = ($rest =~ /^(.*? |)([a-z0-9]+|'s)(-|\xE2\x80[\x93\x94])([a-z0-9].*)$/i)) {
|
|
$new_s .= "$pre$left";
|
|
if ($ht{LC_SPLIT_X_DASH}->{($caller->normalize_punctuation(lc $left))}) {
|
|
$new_s .= " \@" . $dash . "\@ ";
|
|
} else {
|
|
$new_s .= $dash;
|
|
}
|
|
$rest = $post;
|
|
}
|
|
$new_s .= $rest;
|
|
$s = $new_s;
|
|
|
|
|
|
$rest = $s;
|
|
$new_s = "";
|
|
if (($pre, $left, $dash, $right, $post) = ($rest =~ /^(.*?[- ]|)([a-z]+)([-\/]|\xE2\x80[\x93\x94])([a-z]+)([- ].*|)$/i)) {
|
|
}
|
|
while (($pre, $left, $dash, $right, $post) = ($rest =~ /^(.*?[-\/ ]|)([a-z]+)((?: \@)?(?:[-\/]|\xE2\x80[\x93\x94])(?:\@ )?)([a-z]+)([-\/ ].*|)$/i)) {
|
|
$x = $caller->normalize_punctuation(lc ($left . $dash . $right));
|
|
if ($ht{LC_SPLIT}->{($caller->normalize_punctuation(lc ($left . $dash . $right)))}) {
|
|
$pre =~ s/([-\/])$/ \@$1\@ /;
|
|
$post =~ s/^([-\/])/ \@$1\@ /;
|
|
$dash = $caller->normalize_punctuation($dash);
|
|
$new_s .= "$pre$left";
|
|
$new_s .= " \@" . $dash . "\@ ";
|
|
$new_s .= $right;
|
|
$rest = $post;
|
|
} elsif ($pre =~ /[-\/]$/) {
|
|
$new_s .= $pre;
|
|
$rest = "$left$dash$right$post";
|
|
} else {
|
|
$new_s .= "$pre$left";
|
|
$rest = "$dash$right$post";
|
|
}
|
|
}
|
|
$new_s .= $rest;
|
|
$s = $new_s;
|
|
|
|
|
|
$rest = $s;
|
|
$new_s = "";
|
|
while (($pre, $dash, $right, $post) = ($rest =~ /^(.*?) \@(-|\xE2\x80[\x93\x94])\@ ([a-z0-9]+)( .*|)$/i)) {
|
|
$new_s .= $pre;
|
|
if ($ht{LC_DO_NOT_SPLIT_DASH_X}->{($caller->normalize_punctuation(lc $right))}) {
|
|
$new_s .= $dash;
|
|
} else {
|
|
$new_s .= " \@" . $dash . "\@ ";
|
|
}
|
|
$new_s .= $right;
|
|
$rest = $post;
|
|
}
|
|
$new_s .= $rest;
|
|
$s = $new_s;
|
|
|
|
|
|
$rest = $s;
|
|
$new_s = "";
|
|
while (($pre, $left, $dash, $post) = ($rest =~ /^(.*? |)([a-z0-9]+) \@([-\/]|\xE2\x80[\x93\x94]])\@ ([a-z0-9].*)$/i)) {
|
|
$new_s .= "$pre$left";
|
|
if ($ht{LC_DO_NOT_SPLIT_X_DASH}->{($caller->normalize_punctuation(lc $left))}) {
|
|
$new_s .= $dash;
|
|
} else {
|
|
$new_s .= " \@" . $dash . "\@ ";
|
|
}
|
|
$rest = $post;
|
|
}
|
|
$new_s .= $rest;
|
|
$s = $new_s;
|
|
|
|
|
|
$rest = $s;
|
|
$new_s = "";
|
|
while (($pre, $left, $dash, $right, $post) = ($rest =~ /^(.*? |)([a-z0-9]+) \@([-\/]|\xE2\x80[\x93\x94]])\@ ([a-z0-9]+)( .*|)$/i)) {
|
|
$new_s .= "$pre$left";
|
|
if ($ht{LC_DO_NOT_SPLIT}->{($caller->normalize_punctuation(lc ($left . $dash . $right)))}) {
|
|
$new_s .= $dash;
|
|
} else {
|
|
$new_s .= " \@" . $dash . "\@ ";
|
|
}
|
|
$new_s .= $right;
|
|
$rest = $post;
|
|
}
|
|
$new_s .= $rest;
|
|
$s = $new_s;
|
|
|
|
|
|
$rest = $s;
|
|
$new_s = "";
|
|
while (($pre, $left, $dash_group1, $dash1, $middle, $dash_group2, $dash2, $right, $post) = ($rest =~ /^(.*? |)([a-z0-9]+)((?: \@)?([-\/]|\xE2\x80[\x93\x94]])(?:\@ )?)([a-z0-9]+)((?: \@)?([-\/]|\xE2\x80[\x93\x94]])(?:\@ )?)([a-z0-9]+)( .*|)$/i)) {
|
|
$new_s .= "$pre$left";
|
|
if ($ht{LC_DO_NOT_SPLIT}->{($caller->normalize_punctuation(lc ($left . $dash1 . $middle . $dash2 . $right)))}) {
|
|
$new_s .= $dash1;
|
|
} else {
|
|
$new_s .= $dash_group1;
|
|
}
|
|
$new_s .= $middle;
|
|
if ($ht{LC_DO_NOT_SPLIT}->{($caller->normalize_punctuation(lc ($left . $dash1 . $middle . $dash2 . $right)))}) {
|
|
$new_s .= $dash2;
|
|
} else {
|
|
$new_s .= $dash_group2;
|
|
}
|
|
$new_s .= $right;
|
|
$rest = $post;
|
|
}
|
|
$new_s .= $rest;
|
|
$s = $new_s;
|
|
|
|
$s =~ s/ +/ /g;
|
|
}
|
|
print "Point S: $s\n" if $local_verbose;
|
|
|
|
$s =~ s/_DONTBREAK_//g;
|
|
$s =~ s/( )(ark|ill|mass|miss|wash|GA|LA|MO|OP|PA|VA|VT)(\.)( )/$1$2 $3$4/g;
|
|
print "Point T: $s\n" if $local_verbose;
|
|
$s = $caller->restore_urls_x045_guarded_string($s);
|
|
$s = $caller->restore_xml_tags_x0123_guarded_string($s);
|
|
print "Point U: $s\n" if $local_verbose;
|
|
$s =~ s/(https?|ftp)\s*(:)\s*(\/\/)/$1$2$3/gi;
|
|
$s =~ s/\b(mailto)\s*(:)\s*([a-z])/$1$2$3/gi;
|
|
$s =~ s/(\d)\s*(:)\s*([0-5]\d[^0-9])/$1$2$3/gi;
|
|
print "Point V: $s\n" if $local_verbose;
|
|
$s =~ s/(5\xE2\x80\xB2-[ACGT]+)\s*(-|\xE2\x80[\x93\x94])\s*(3\xE2\x80\xB2)/$1$2$3/g;
|
|
$s =~ s/ (etc) \. / $1. /g;
|
|
print "Point W: $s\n" if $local_verbose;
|
|
$s = $caller->repair_separated_periods($s);
|
|
print "Point X: $s\n" if $local_verbose;
|
|
$s =~ s/^\s+//;
|
|
$s =~ s/\s+$//;
|
|
$s = "$pre$s$post" if defined($pre) && defined($post);
|
|
$s =~ s/ +/ /g;
|
|
print "Point Y: $s\n" if $local_verbose;
|
|
|
|
return $s;
|
|
}
|
|
|
|
sub tokenize_plus_for_noisy_text {
|
|
local($caller, $s, *ht, $control) = @_;
|
|
|
|
$control = "" unless defined($control);
|
|
my $pre;
|
|
my $code;
|
|
my $post;
|
|
$s = " $core " if ($pre,$core,$post) = ($s =~ /^(\s*)(.*?)(\s*)$/i);
|
|
foreach $i ((1 .. 2)) {
|
|
$s =~ s/ ([A-Z][a-z]+'?[a-z]+)(-) / $1 $2 /gi;
|
|
$s =~ s/ (\d+(?:\.\d+)?)(-|:-|:|_|\.|'|;)([A-Z][a-z]+'?[a-z]+|[A-Z]{3,}) / $1 $2 $3 /gi;
|
|
$s =~ s/ (\d+(?:\.\d+)?)(')([A-Za-z]{3,}) / $1 $2 $3 /gi;
|
|
$s =~ s/ (-|:-|:|_|\.)([A-Z][a-z]+'?[a-z]+|[A-Z]{3,}) / $1 $2 /gi;
|
|
$s =~ s/ ((?:[A-Z]\.[A-Z]|[A-Z]|Amb|Col|Dr|Eng|Gen|Inj|Lt|Maj|Md|Miss|Mr|Mrs|Ms|Pres|Prof|Sen)\.)([A-Z][a-z]+|[A-Z]{2,}) / $1 $2 /gi;
|
|
$s =~ s/ (\d+)(,)([a-z]{3,}) / $1 $2 $3 /gi;
|
|
$s =~ s/ (%)(\d+(?:\.\d+)?) / $1 $2 /gi;
|
|
$s =~ s/ ([A-Za-z][a-z]{3,}\d*)([.,\/]|:\()([A-Za-z][a-z]{3,}|[A-Z]{3,}) / $1 $2 $3 /gi;
|
|
$s =~ s/ ([A-Za-z]{3,}\d*?|[A-Za-z]+'[A-Za-z]+)([,\/]|:\()([A-Za-z]{3,}|[A-Za-z]+'[A-Za-z]+) / $1 $2 $3 /gi;
|
|
$s =~ s/ (\d[0-9.,]*\d)(;)([a-z]+) / $1 $2 $3 /gi;
|
|
}
|
|
$s =~ s/^\s+//;
|
|
$s =~ s/\s+$//;
|
|
$s = "$pre$s$post" if defined($pre) && defined($post);
|
|
return $s;
|
|
}
|
|
|
|
|
|
|
|
my $abbrev_s = "etc.|e.g.|i.e.|U.K.|S.p.A.|A.F.P.";
|
|
my @abbrevs = split(/\|/, $abbrev_s);
|
|
my @exp_abbrevs = ();
|
|
foreach $abbrev (@abbrevs) {
|
|
if (($core,$period) = ($abbrev =~ /^(.*?)(\.|)$/)) {
|
|
$core =~ s/\./\\s*\\.\\s*/g;
|
|
$abbrev = $core;
|
|
$abbrev .= "\\b" if $abbrev =~ /[a-z]$/i;
|
|
$abbrev .= "(?:\\s*\\.|)" if $period;
|
|
push(@exp_abbrevs, $abbrev);
|
|
}
|
|
}
|
|
my $exp_abbrev_s = join("|", @exp_abbrevs);
|
|
|
|
sub repair_separated_periods {
|
|
local($caller,$s) = @_;
|
|
|
|
|
|
my $result = "";
|
|
while (($pre, $abbrev, $post) = ($s =~ /^(.*? |)($exp_abbrev_s)(.*)$/)) {
|
|
$abbrev =~ s/ //g;
|
|
$abbrev .= "." unless $abbrev =~ /\.$/;
|
|
$result .= "$pre$abbrev ";
|
|
$s = $post;
|
|
}
|
|
$result .= $s;
|
|
$result =~ s/ +/ /g;
|
|
return $result;
|
|
}
|
|
|
|
|
|
sub fix_tokenize {
|
|
local($caller,$s) = @_;
|
|
|
|
|
|
$s =~ s/(\d)\:(\d)/$1 \@:\@ $2/g;
|
|
|
|
|
|
$s =~ s/(^|\s)0+(\d)/$1$2/g;
|
|
|
|
|
|
$s =~ s/associatedpress/associated press/g;
|
|
|
|
|
|
$s =~ s/hong_kong/hong kong/g;
|
|
$s =~ s/united_states/united states/g;
|
|
|
|
return $s;
|
|
}
|
|
|
|
sub de_mt_tokenize {
|
|
local($caller,$s) = @_;
|
|
|
|
$s =~ s/\s+\@([-:\/])/$1/g;
|
|
$s =~ s/([-:\/])\@\s+/$1/g;
|
|
$s =~ s/\s+\/\s+/\//g;
|
|
return $s;
|
|
}
|
|
|
|
sub surface_forms {
|
|
local($caller,$pe,$modp) = @_;
|
|
|
|
$sem = $pe->sem;
|
|
$surf = $pe->surf;
|
|
$synt = $pe->synt;
|
|
$value = $pe->value;
|
|
$gloss = $pe->gloss;
|
|
|
|
if ($sem eq "integer") {
|
|
return ($gloss) if ($gloss =~ /several/) && !($value =~ /\S/);
|
|
print STDERR "Warning: $value not an integer\n" unless $value =~ /^\d+(e\+\d+)?$/;
|
|
if ($pe->get("reliable") =~ /sequence of digits/) {
|
|
$english = $value;
|
|
$english = "$prefix $english" if $prefix = $pe->get("prefix");
|
|
@result = ($english);
|
|
} else {
|
|
@result = $caller->q_number_surface_forms($pe);
|
|
}
|
|
} elsif ($sem eq "decimal number") {
|
|
@result = $caller->q_number_surface_forms($pe);
|
|
} elsif ($sem =~ /(integer|decimal number) range/) {
|
|
@result = $caller->number_range_surface_forms($pe);
|
|
} elsif ($sem eq "ordinal") {
|
|
if ($pe->get("definite")) {
|
|
$exclude_adverbials_p = 1;
|
|
} elsif (defined($chinesePM) && ($hao = $chinesePM->e2c("hao-day"))
|
|
&& ($gc = $chinesePM->e2c("generic counter"))) {
|
|
$exclude_adverbials_p = ($surf =~ /($hao|$gc)$/);
|
|
} else {
|
|
$exclude_adverbials_p = 1;
|
|
}
|
|
@result = $caller->ordinal_surface_forms($pe->get("ordvalue") || $pe->value,0,$exclude_adverbials_p, $pe);
|
|
} elsif ($sem eq "fraction") {
|
|
@result = $caller->fraction_surface_forms($pe,$modp);
|
|
} elsif ($sem =~ /monetary quantity/) {
|
|
@result = $caller->currency_surface_forms($pe);
|
|
} elsif ($sem =~ /occurrence quantity/) {
|
|
@result = $caller->occurrence_surface_forms($pe,$modp);
|
|
} elsif ($sem =~ /score quantity/) {
|
|
@result = $caller->score_surface_forms($pe);
|
|
} elsif ($sem =~ /age quantity/) {
|
|
@result = $caller->age_surface_forms($pe, $modp);
|
|
} elsif ($sem =~ /quantity/) {
|
|
@result = $caller->quantity_surface_forms($pe,$modp);
|
|
} elsif ($sem eq "percentage") {
|
|
@result = $caller->percent_surface_forms($pe,$modp);
|
|
} elsif ($sem eq "percentage range") {
|
|
if ($gloss =~ /^and /) {
|
|
@result = ($gloss);
|
|
} else {
|
|
@result = ($gloss, "by $gloss", "of $gloss");
|
|
}
|
|
} elsif ($sem =~ /^(month of the year|month\+year|year)$/) {
|
|
if ($synt eq "pp") {
|
|
@result = ($gloss);
|
|
} elsif ($gloss =~ /^the (beginning|end) of/) {
|
|
@result = ($gloss, "at $gloss");
|
|
} elsif ($gloss =~ /^(last|this|current|next)/) {
|
|
@result = ($gloss);
|
|
} else {
|
|
|
|
@result = ($gloss, "in $gloss");
|
|
}
|
|
} elsif ($sem =~ /date(\+year)?$/) {
|
|
@result = $caller->date_surface_forms($pe,$modp);
|
|
} elsif ($sem =~ /year range\b.*\b(decade|century)$/) {
|
|
@result = $caller->decade_century_surface_forms($pe);
|
|
} elsif ($sem eq "day of the month") {
|
|
@result = $caller->day_of_the_month_surface_forms($pe);
|
|
} elsif ($sem =~ /period of the day\+day of the week/) {
|
|
@result = ($gloss);
|
|
push(@result, "on $gloss") if $gloss =~ /^the night/;
|
|
} elsif ($sem =~ /day of the week/) {
|
|
@result = $caller->day_of_the_week_surface_forms($pe);
|
|
} elsif ($sem =~ /^(time)$/) {
|
|
if ($gloss =~ /^at /) {
|
|
@result = ($gloss);
|
|
} else {
|
|
@result = ($gloss, "at $gloss");
|
|
}
|
|
} elsif ($sem =~ /^date range$/) {
|
|
if ($synt eq "pp") {
|
|
@result = ($gloss);
|
|
} elsif ($pe->get("between")) {
|
|
$b_gloss = "between $gloss";
|
|
$b_gloss =~ s/-/ and /;
|
|
@result = ($b_gloss, $gloss, "from $gloss");
|
|
} else {
|
|
@result = ($gloss, "from $gloss");
|
|
}
|
|
} elsif ($sem =~ /^date enumeration$/) {
|
|
if ($synt eq "pp") {
|
|
@result = ($gloss);
|
|
} else {
|
|
@result = ($gloss, "on $gloss");
|
|
}
|
|
} elsif ($pe->get("unknown-in-pc")) {
|
|
@result = ();
|
|
foreach $unknown_pos_en (split(/;;/, $pe->get("unknown-pos-en-list"))) {
|
|
($engl) = ($unknown_pos_en =~ /^[^:]+:[^:]+:(.*)$/);
|
|
push(@result, $engl) if defined($engl) && ! $util->member($engl, @result);
|
|
}
|
|
@result = ($gloss) unless @result;
|
|
} elsif (($sem =~ /\b(name|unknown)\b/) && (($en_s = $pe->get("english")) =~ /[a-z]/i)) {
|
|
@result = split(/\s*\|\s*/, $en_s);
|
|
} elsif (($sem =~ /^proper\b/) && (($en_s = $pe->get("english")) =~ /[a-z]/i)) {
|
|
@result = split(/\s*\|\s*/, $en_s);
|
|
} else {
|
|
@result = ($gloss);
|
|
}
|
|
|
|
if (($sem =~ /^(date\+year|month\+year|year)$/)
|
|
&& ($year = $pe->get("year"))
|
|
&& ($year =~ /^\d\d$/)
|
|
&& (@extend_years = @{$english_entity_style_ht{"ExtendYears"}})
|
|
&& ($#extend_years == 1)
|
|
&& ($extended_year_start = $extend_years[0])
|
|
&& ($extended_year_end = $extend_years[1])
|
|
&& ($extended_year_start <= $extended_year_end)
|
|
&& ($extended_year_start + 99 >= $extended_year_end)
|
|
&& ($extended_year_start =~ /^\d\d\d\d$/)
|
|
&& ($extended_year_end =~ /^\d\d\d\d$/)) {
|
|
$century1 = substr($extended_year_start, 0, 2);
|
|
$century2 = substr($extended_year_end, 0, 2);
|
|
$exp_year1 = "$century1$year";
|
|
$exp_year2 = "$century2$year";
|
|
if (($extended_year_start <= $exp_year1) && ($exp_year1 <= $extended_year_end)) {
|
|
$exp_year = $exp_year1;
|
|
} elsif (($extended_year_start <= $exp_year2) && ($exp_year2 <= $extended_year_end)) {
|
|
$exp_year = $exp_year2;
|
|
} else {
|
|
$exp_year = "";
|
|
}
|
|
if ($exp_year) {
|
|
@new_glosses = ();
|
|
foreach $old_gloss (@result) {
|
|
$new_gloss = $old_gloss;
|
|
$new_gloss =~ s/\b$year$/$exp_year/;
|
|
push (@new_glosses, $new_gloss) unless $new_gloss eq $old_gloss;
|
|
}
|
|
push (@result, @new_glosses);
|
|
}
|
|
}
|
|
|
|
|
|
@tokenize_list = @{$english_entity_style_ht{"Tokenize"}};
|
|
$tokenize_p = 1 if $util->member("yes", @tokenize_list)
|
|
|| $util->member("all", @tokenize_list);
|
|
$dont_tokenize_p = 1 if $util->member("no", @tokenize_list)
|
|
|| $util->member("all", @tokenize_list);
|
|
if ($tokenize_p) {
|
|
@new_result = ();
|
|
foreach $item (@result) {
|
|
$t_item = $caller->tokenize($item, *dummy_ht);
|
|
push(@new_result, $item) if $dont_tokenize_p && ($item ne $t_item);
|
|
push(@new_result, $t_item);
|
|
}
|
|
@result = @new_result;
|
|
}
|
|
|
|
|
|
@case_list = @{$english_entity_style_ht{"Case"}};
|
|
$lower_case_p = $util->member("lower", @case_list)
|
|
|| $util->member("all", @case_list);
|
|
$reg_case_p = $util->member("regular", @case_list)
|
|
|| $util->member("all", @case_list);
|
|
if ($lower_case_p) {
|
|
@new_result = ();
|
|
foreach $item (@result) {
|
|
$l_item = "\L$item";
|
|
push(@new_result, $item) if $reg_case_p && ($item ne $l_item);
|
|
push(@new_result, $l_item) unless $util->member($l_item, @new_result);
|
|
}
|
|
@result = @new_result;
|
|
}
|
|
|
|
|
|
return @result;
|
|
}
|
|
|
|
sub case_list {
|
|
return @{$english_entity_style_ht{"Case"}};
|
|
}
|
|
|
|
sub right_cased_list {
|
|
local($caller, $word) = @_;
|
|
|
|
@case_list = @{$english_entity_style_ht{"Case"}};
|
|
|
|
@right_cased_core_list = ();
|
|
push(@right_cased_core_list, $word)
|
|
if ($util->member("regular", @case_list) || $util->member("all", @case_list))
|
|
&& ! $util->member($word, @right_cased_core_list);
|
|
push(@right_cased_core_list, lc $word)
|
|
if ($util->member("lower", @case_list) || $util->member("all", @case_list))
|
|
&& ! $util->member(lc $word, @right_cased_core_list);
|
|
|
|
return @right_cased_core_list;
|
|
}
|
|
|
|
sub string2surf_forms {
|
|
local($caller, $text, $lang, $alt_sep) = @_;
|
|
|
|
$alt_sep = " | " unless defined($alt_sep);
|
|
$lang = "zh" unless defined($lang);
|
|
|
|
if ($lang eq "zh") {
|
|
@pes = $chinesePM->parse_entities_in_string($text);
|
|
$n = $#pes + 1;
|
|
|
|
@pes = $chinesePM->select_reliable_entities(@pes);
|
|
my @res_surf_forms_copy = $caller->reliable_pes2surf_forms($alt_sep, @pes);
|
|
return @res_surf_forms_copy;
|
|
} else {
|
|
return ();
|
|
}
|
|
}
|
|
|
|
sub reliable_pe2surf_forms {
|
|
local($caller, $pe, $parent_reliant_suffices_p) = @_;
|
|
|
|
$parent_reliant_suffices_p = 0 unless defined($parent_reliant_suffices_p);
|
|
if ((defined($r = $pe->get("reliable")) && $r)
|
|
|| ($parent_reliant_suffices_p && ($parent_pe = $pe->get("parent")) &&
|
|
$parent_pe->get("reliable"))) {
|
|
@surf_forms = $caller->surface_forms($pe);
|
|
if ((($pe->sem =~ /quantity( range)?$/) && !($pe->sem =~ /monetary quantity/))
|
|
|| ($util->member($pe->sem, "percentage","fraction"))) {
|
|
foreach $mod_form ($caller->surface_forms($pe, 1)) {
|
|
push(@surf_forms, $mod_form) unless $util->member($mod_form, @surf_forms);
|
|
}
|
|
}
|
|
return @surf_forms;
|
|
}
|
|
return ();
|
|
}
|
|
|
|
sub reliable_pe2surf_form {
|
|
local($caller, $alt_sep, $pe) = @_;
|
|
|
|
if (@surf_forms = $caller->reliable_pe2surf_forms($pe)) {
|
|
return $pe->surf . " == " . join($alt_sep, @surf_forms);
|
|
} else {
|
|
return "";
|
|
}
|
|
}
|
|
|
|
sub reliable_pes2surf_forms {
|
|
local($caller, $alt_sep, @pes) = @_;
|
|
|
|
my @res_surf_forms = ();
|
|
foreach $pe (@pes) {
|
|
if ($new_surf_form = $caller->reliable_pe2surf_form($alt_sep, $pe)) {
|
|
push(@res_surf_forms, $new_surf_form);
|
|
}
|
|
}
|
|
return @res_surf_forms;
|
|
}
|
|
|
|
sub string_contains_ascii_letter {
|
|
local($caller,$string) = @_;
|
|
return $string =~ /[a-zA-Z]/;
|
|
}
|
|
|
|
sub string_starts_w_ascii_letter {
|
|
local($caller,$string) = @_;
|
|
return $string =~ /^[a-zA-Z]/;
|
|
}
|
|
|
|
sub en_lex_bin {
|
|
local($caller, $word) = @_;
|
|
|
|
$word =~ s/\s+//g;
|
|
$word =~ s/[-_'\/]//g;
|
|
$word =~ tr/A-Z/a-z/;
|
|
return "digit" if $word =~ /^\d/;
|
|
return "special" unless $word =~ /^[a-z]/;
|
|
return substr($word, 0, 1);
|
|
}
|
|
|
|
sub skeleton_bin {
|
|
local($caller, $sk_bin_control, $word) = @_;
|
|
|
|
$word =~ s/\s+//g;
|
|
$word =~ s/[-_'\/]//g;
|
|
$word =~ tr/A-Z/a-z/;
|
|
return "E" unless $word;
|
|
if ($sk_bin_control =~ /^v1/i) {
|
|
return $word if length($word) <= 2;
|
|
return substr($word, 0, 3) if $word =~ /^(b|f[lnrt]|gr|j[nr]|k|l[nt]|m|n[kmst]|r[knst]|s|t)/;
|
|
return substr($word, 0, 2);
|
|
} elsif ($sk_bin_control =~ /d6f$/) {
|
|
return $word if length($word) <= 6;
|
|
return substr($word, 0, 6);
|
|
} elsif ($sk_bin_control =~ /d5f$/) {
|
|
return $word if length($word) <= 5;
|
|
return substr($word, 0, 5);
|
|
} elsif ($sk_bin_control =~ /d4f$/) {
|
|
return $word if length($word) <= 4;
|
|
return substr($word, 0, 4);
|
|
} else {
|
|
return $word if length($word) <= 4;
|
|
return substr($word, 0, 5) if $word =~ /^(bnts|brnt|brst|brtk|brtn|brts|frst|frts|klts|kntr|knts|krst|krtn|krts|ksks|kstr|lktr|ntrs|sbrt|skrt|sntr|strn|strt|trns|trts|ts)/;
|
|
return substr($word, 0, 4);
|
|
}
|
|
}
|
|
|
|
sub skeleton_bin_sub_dir {
|
|
local($caller, $sk_bin_control, $skeleton_bin) = @_;
|
|
|
|
$sk_bin_control = "v1" unless defined($sk_bin_control);
|
|
return "" if $sk_bin_control =~ /^v1/i;
|
|
if ($sk_bin_control =~ /^2d4d\df$/) {
|
|
return "SH/SHOR" if (length($skeleton_bin) < 2);
|
|
return substr($skeleton_bin, 0, 2) . "/" . substr($skeleton_bin, 0, 2) . "SH" if (length($skeleton_bin) < 4);
|
|
return substr($skeleton_bin, 0, 2) . "/" . substr($skeleton_bin, 0, 4);
|
|
} elsif ($sk_bin_control =~ /^2d3d\df$/) {
|
|
return "SH/SHO" if (length($skeleton_bin) < 2);
|
|
return substr($skeleton_bin, 0, 2) . "/" . substr($skeleton_bin, 0, 2) . "S" if (length($skeleton_bin) < 3);
|
|
return substr($skeleton_bin, 0, 2) . "/" . substr($skeleton_bin, 0, 3);
|
|
}
|
|
$bin3 = "ts";
|
|
return "SH" if (length($skeleton_bin) < 2) || ($skeleton_bin =~ /^($bin3)$/);
|
|
return substr($skeleton_bin, 0, 3) if $skeleton_bin =~ /^($bin3)/;
|
|
return substr($skeleton_bin, 0, 2);
|
|
}
|
|
|
|
sub en_words_and_counts_matching_skeletons {
|
|
local($caller, $sk_bin_version, @skeletons) = @_;
|
|
|
|
return () unless @skeletons;
|
|
|
|
@rem_skeletons = sort @skeletons;
|
|
$previous_skeleton = "";
|
|
$current_skeleton = shift @rem_skeletons;
|
|
@list = ($current_skeleton);
|
|
@lists = ();
|
|
|
|
$current_bin = "";
|
|
while ($current_skeleton) {
|
|
unless ($current_skeleton eq $previous_skeleton) {
|
|
$current_skeleton_bin = $caller->skeleton_bin($sk_bin_version, $current_skeleton);
|
|
unless ($current_skeleton_bin eq $current_bin) {
|
|
|
|
close(IN) if $current_bin;
|
|
$current_bin = $current_skeleton_bin;
|
|
$current_bin_subdir
|
|
= $caller->skeleton_bin_sub_dir($sk_bin_version, $current_bin);
|
|
if ($current_bin_subdir) {
|
|
$en_skeleton_file = File::Spec->catfile($english_resources_skeleton_dir,
|
|
$current_bin_subdir,
|
|
"$current_bin.txt");
|
|
} else {
|
|
$en_skeleton_file = File::Spec->catfile($english_resources_skeleton_dir,
|
|
"$current_bin.txt");
|
|
}
|
|
|
|
if (open(IN, $en_skeleton_file)) {
|
|
$en_skeleton_file_exists = 1;
|
|
} else {
|
|
$en_skeleton_file_exists = 0;
|
|
print STDERR "Can't open $en_skeleton_file (Point A)\n";
|
|
}
|
|
}
|
|
$previous_skeleton = $current_skeleton;
|
|
}
|
|
$_ = <IN> if $en_skeleton_file_exists;
|
|
unless ($en_skeleton_file_exists && defined($_)) {
|
|
push(@lists, join(' ; ', @list));
|
|
if (@rem_skeletons) {
|
|
$current_skeleton = shift @rem_skeletons;
|
|
@list = ($current_skeleton);
|
|
} else {
|
|
$current_skeleton = "";
|
|
}
|
|
next;
|
|
}
|
|
($skeleton) = ($_ =~ /^(\S+)\t/);
|
|
next unless defined($skeleton);
|
|
$skeletons_match_p = $caller->skeletons_match_p($skeleton, $current_skeleton);
|
|
next if ($skeleton lt $current_skeleton) && ! $skeletons_match_p;
|
|
if ($skeletons_match_p) {
|
|
($token, $count) = ($_ =~ /^\S+\t(\S|\S[-' a-zA-Z]*\S)\t(\d+)\s*$/);
|
|
push(@list, "$token : $count") if defined($token) && defined($count);
|
|
} else {
|
|
while ($current_skeleton lt $skeleton) {
|
|
push(@lists, join(' ; ', @list));
|
|
unless (@rem_skeletons) {
|
|
close(IN) if $current_bin;
|
|
$current_skeleton = "";
|
|
last;
|
|
}
|
|
$current_skeleton = shift @rem_skeletons;
|
|
@list = ($current_skeleton);
|
|
}
|
|
if ($caller->skeletons_match_p($skeleton, $current_skeleton)) {
|
|
($token, $count) = ($_ =~ /^\S+\t(\S|\S[-' a-zA-Z]*\S)\t(\d+)\s*$/);
|
|
push(@list, "$token : $count") if defined($token) && defined($count);
|
|
}
|
|
}
|
|
}
|
|
close(IN) if $current_bin;
|
|
return @lists;
|
|
}
|
|
|
|
sub skeletons_match_p {
|
|
|
|
local($caller, $skeleton1, $skeleton2, $max) = @_;
|
|
|
|
return 1 if $skeleton1 eq $skeleton2;
|
|
|
|
$max = 5 unless defined($max);
|
|
if ((length($skeleton1) > length($skeleton2)) && (length($skeleton2) == $max)) {
|
|
return ($skeleton1 =~ /^$skeleton2/) ? 1 : 0;
|
|
} elsif ((length($skeleton2) > length($skeleton1)) && (length($skeleton1) == $max)) {
|
|
return ($skeleton2 =~ /^$skeleton1/) ? 1 : 0;
|
|
} else {
|
|
return 0;
|
|
}
|
|
}
|
|
|
|
sub token_weird_or_too_long {
|
|
local($caller, *WARNING_FH, $token) = @_;
|
|
|
|
$lc_token = lc $token;
|
|
$norm_token = $lc_token;
|
|
$norm_token =~ s/[-' ,]//g;
|
|
$snippet4_5 = "";
|
|
$snippet4_5 = substr($norm_token, 4, 2) if length($norm_token) >= 10;
|
|
$snippet4_6 = "";
|
|
$snippet4_6 = substr($norm_token, 4, 3) if length($norm_token) >= 10;
|
|
if (($norm_token =~ /(kkk|vvv|www|xxx|yyy|zzz)/) ||
|
|
($norm_token =~ /[acgt]{15,}/) ||
|
|
($snippet4_5 && ($norm_token =~ /($snippet4_5){5,}/)) ||
|
|
($snippet4_6 && ($norm_token =~ /($snippet4_6){4,}/)) ||
|
|
($norm_token =~ /[bcdfghjklmnpqrstvwxz]{8,}/) ||
|
|
($token =~ /(DDD)/) ||
|
|
(($lc_token =~ /fff/) && ! ($lc_token =~ /schifff/))) {
|
|
print WARNING_FH "skipping (WEIRD): $_";
|
|
return 1;
|
|
}
|
|
if ((length($norm_token) >= 50) ||
|
|
((length($norm_token) >= 28)
|
|
|
|
|
|
&& ! ($norm_token =~ /entwicklung/)
|
|
&& ! ($norm_token =~ /fabrik/)
|
|
&& ! ($norm_token =~ /finanz/)
|
|
&& ! ($norm_token =~ /forschung/)
|
|
&& ! ($norm_token =~ /geschwindigkeit/)
|
|
&& ! ($norm_token =~ /gesundheit/)
|
|
&& ! ($norm_token =~ /gewohnheit/)
|
|
&& ! ($norm_token =~ /schaft/)
|
|
&& ! ($norm_token =~ /schifffahrt/)
|
|
&& ! ($norm_token =~ /sicherheit/)
|
|
&& ! ($norm_token =~ /vergangen/)
|
|
&& ! ($norm_token =~ /versicherung/)
|
|
&& ! ($norm_token =~ /unternehmen/)
|
|
&& ! ($norm_token =~ /verwaltung/)
|
|
|
|
|
|
&& ! ($norm_token =~ /aktiebolag/)
|
|
&& ! ($norm_token =~ /aktieselskab/)
|
|
&& ! ($norm_token =~ /ontwikkeling/)
|
|
|
|
|
|
&& ! ($norm_token =~ /phetamine/)
|
|
&& ! ($norm_token =~ /ethyl/)
|
|
|
|
|
|
&& ! ($norm_token =~ /^pneumonaultramicroscopicsilicovolcanoconios[ei]s$/)
|
|
|
|
|
|
&& ! ($norm_token =~ /PriceWaterhouse/)
|
|
)) {
|
|
print WARNING_FH "skipping (TOO LONG): $_";
|
|
return 1;
|
|
}
|
|
return 0;
|
|
}
|
|
|
|
sub xml_de_accent {
|
|
local($caller, $string) = @_;
|
|
|
|
|
|
|
|
$string =~ s/\[2-7];/A/g;
|
|
$string =~ s/\Æ/Ae/g;
|
|
$string =~ s/\Ç/C/g;
|
|
$string =~ s/\[0-3];/E/g;
|
|
$string =~ s/\[4-7];/I/g;
|
|
$string =~ s/\Ð/Dh/g;
|
|
$string =~ s/\Ñ/N/g;
|
|
$string =~ s/\[0-4];/O/g;
|
|
$string =~ s/\Ø/O/g;
|
|
$string =~ s/\[7-9];/U/g;
|
|
$string =~ s/\Ü/U/g;
|
|
$string =~ s/\Ý/Y/g;
|
|
$string =~ s/\Þ/Th/g;
|
|
|
|
$string =~ s/\ß/ss/g;
|
|
$string =~ s/\[4-9];/a/g;
|
|
$string =~ s/\æ/ae/g;
|
|
$string =~ s/\ç/c/g;
|
|
$string =~ s/\[2-5];/e/g;
|
|
$string =~ s/\[6-9];/i/g;
|
|
$string =~ s/\ð/dh/g;
|
|
$string =~ s/\ñ/n/g;
|
|
$string =~ s/\[2-6];/o/g;
|
|
$string =~ s/\ø/o/g;
|
|
$string =~ s/\ù/u/g;
|
|
$string =~ s/\[0-2];/u/g;
|
|
$string =~ s/\ý/y/g;
|
|
$string =~ s/\þ/th/g;
|
|
$string =~ s/\ÿ/y/g;
|
|
$string =~ s/\xE2\x80\x99/'/g;
|
|
|
|
return $string;
|
|
}
|
|
|
|
sub de_accent {
|
|
local($caller, $string) = @_;
|
|
|
|
|
|
|
|
$string =~ s/\xC3[\x80-\x85]/A/g;
|
|
$string =~ s/\xC3\x86/Ae/g;
|
|
$string =~ s/\xC3\x87/C/g;
|
|
$string =~ s/\xC3[\x88-\x8B]/E/g;
|
|
$string =~ s/\xC3[\x8C-\x8F]/I/g;
|
|
$string =~ s/\xC3\x90/Dh/g;
|
|
$string =~ s/\xC3\x91/N/g;
|
|
$string =~ s/\xC3[\x92-\x96]/O/g;
|
|
$string =~ s/\xC3\x98/O/g;
|
|
$string =~ s/\xC3[\x99-\x9C]/U/g;
|
|
$string =~ s/\xC3\x9D/Y/g;
|
|
$string =~ s/\xC3\x9E/Th/g;
|
|
|
|
$string =~ s/\xC3\x9F/ss/g;
|
|
$string =~ s/\xC3[\xA0-\xA5]/a/g;
|
|
$string =~ s/\xC3\xA6/ae/g;
|
|
$string =~ s/\xC3\xA7/c/g;
|
|
$string =~ s/\xC3[\xA8-\xAB]/e/g;
|
|
$string =~ s/\xC3[\xAC-\xAF]/i/g;
|
|
$string =~ s/\xC3\xB0/dh/g;
|
|
$string =~ s/\xC3\xB1/n/g;
|
|
$string =~ s/\xC3[\xB2-\xB6]/o/g;
|
|
$string =~ s/\xC3\xB8/o/g;
|
|
$string =~ s/\xC3[\xB9-\xBC]/u/g;
|
|
$string =~ s/\xC3\xBD/y/g;
|
|
$string =~ s/\xC3\xBE/th/g;
|
|
$string =~ s/\xC3\xBF/y/g;
|
|
$string =~ s/\xE2\x80\x99/'/g;
|
|
|
|
return $string;
|
|
}
|
|
|
|
sub common_non_name_cap_p {
|
|
local($caller, $word) = @_;
|
|
return defined($english_ht{(lc $word)}->{COMMON_NON_NAME_CAP});
|
|
}
|
|
|
|
sub language {
|
|
return "English";
|
|
}
|
|
|
|
sub language_id {
|
|
return "en";
|
|
}
|
|
|
|
sub parse_entities_in_string {
|
|
local($caller, $string) = @_;
|
|
|
|
$ParseEntry->set_current_lang("en");
|
|
@pes = $ParseEntry->init_ParseEntry_list($string);
|
|
@pes = $caller->lexical_heuristic(@pes);
|
|
@pes = $caller->base_number_heuristic(@pes);
|
|
|
|
return @pes;
|
|
}
|
|
|
|
sub lexical_heuristic {
|
|
local($caller, @pes) = @_;
|
|
|
|
$i = 0;
|
|
while ($i <= $#pes) {
|
|
$pe = $pes[$i];
|
|
if ($pe->undefined("synt")) {
|
|
if ($pe->surf =~ /^\d+(,\d\d\d)*\.\d+/) {
|
|
$pe->set("synt", "cardinal");
|
|
$pe->set("sem", "decimal number");
|
|
$value = $pe->surf;
|
|
$value =~ s/,//g;
|
|
$pe->set("value", $value);
|
|
} elsif ($pe->surf =~ /^\d+(,\d\d\d)*$/) {
|
|
$pe->set("synt", "cardinal");
|
|
$pe->set("sem", "integer");
|
|
$value = $pe->surf;
|
|
$value =~ s/,//g;
|
|
$pe->set("value", $value);
|
|
} elsif ($pe->surf =~ /^([-",\.;\s:()\/%]|\@[-:\/]\@|[-:\/]\@|\@[-:\/])$/) {
|
|
$pe->set("gloss", $pe->surf);
|
|
$pe->set("synt", "punctuation");
|
|
} else {
|
|
($length,$english) = $caller->find_max_lex_match($i,3,@pes);
|
|
if ($length) {
|
|
if ($length > 1) {
|
|
@slot_value_list = ();
|
|
@children = splice(@pes,$i,$length);
|
|
@roles = $util->list_with_same_elem($length,"lex");
|
|
$pe = $ParseEntry->newParent(*slot_value_list,*children,*roles);
|
|
$pe->set("surf",$english);
|
|
$pe->set("eot",1) if $pe->eot_p;
|
|
splice(@pes,$i,0,$pe);
|
|
} else {
|
|
$pe = $pes[$i];
|
|
}
|
|
$annot_s = $english_annotation_ht{$english};
|
|
$annot_s =~ s/^\s*:+//;
|
|
$annot_s =~ s/^\s+//;
|
|
$annot_s =~ s/\s+$//;
|
|
$annot_s =~ s/#.*$//;
|
|
foreach $annot (split('::', $annot_s)) {
|
|
($slot, $value) = ($annot =~ /^([^:]+):(.*)$/);
|
|
if (defined($slot) && defined($value)) {
|
|
$pe->set($slot, $value);
|
|
}
|
|
$pe->set("sem", "integer") if ($slot eq "synt") && ($value eq "cardinal");
|
|
}
|
|
$pe->set("ord-value", $ord_value)
|
|
if $ord_value = $english_annotation_ht{"_EN_SYNT_"}->{(lc $english)}->{"ordinal"}->{"value"};
|
|
$pe->set("card-value", $card_value)
|
|
if $card_value = $english_annotation_ht{"_EN_SYNT_"}->{(lc $english)}->{"cardinal"}->{"value"};
|
|
}
|
|
}
|
|
}
|
|
$i++;
|
|
}
|
|
return @pes;
|
|
}
|
|
|
|
|
|
sub base_number_heuristic {
|
|
local($caller, @pes) = @_;
|
|
|
|
$i = 0;
|
|
|
|
while ($i <= $#pes) {
|
|
|
|
($head_pe, @pes) =
|
|
$ParseEntry->build_parse_entry("composite number plus","",$i,*pes,
|
|
' :head :($pe->sem eq "integer") && ($pe->value =~ /^[1-9]0$/)',
|
|
'optional:dummy:$pe->surf eq "\@-\@"',
|
|
' :mod :($pe->sem eq "integer") && ($pe->value =~ /^[1-9]$/)');
|
|
if ($head_pe) {
|
|
$value1 = $head_pe->childValue("head");
|
|
$value2 = $head_pe->childValue("mod");
|
|
$head_pe->set("value", $value1 + $value2);
|
|
}
|
|
|
|
($head_pe, @pes) =
|
|
$ParseEntry->build_parse_entry("composite number 1000","",$i,*pes,
|
|
' :mod :(($value1 = $pe->value) =~ /^\d+(.\d+)?$/) && ($value1 < 1000)',
|
|
' :head:($value2 = $pe->value) =~ /^1(000)+$/');
|
|
if ($head_pe) {
|
|
$value1 = $head_pe->childValue("mod");
|
|
$value2 = $head_pe->childValue("head");
|
|
$head_pe->set("value", $value1 * $value2);
|
|
}
|
|
|
|
($head_pe, @pes) =
|
|
$ParseEntry->build_parse_entry("composite ordinal","",$i,*pes,
|
|
' :mod :($pe->sem eq "integer") && ($pe->value =~ /^[1-9]0$/)',
|
|
'optional:dummy:$pe->surf eq "\@-\@"',
|
|
' :head :$pe->get("ord-value") =~ /^[1-9]$/');
|
|
if ($head_pe) {
|
|
$value1 = $head_pe->childSlot("head", "ord-value");
|
|
$value2 = $head_pe->childValue("mod");
|
|
$head_pe->set("value", $value1 + $value2);
|
|
}
|
|
$i++;
|
|
}
|
|
|
|
return @pes;
|
|
}
|
|
|
|
sub find_max_lex_match {
|
|
local($caller,$start,$maxlength,@pes) = @_;
|
|
|
|
while ($maxlength > 0) {
|
|
if (($english = $util->pes_subseq_surf($start,$maxlength,"en",@pes))
|
|
&& defined($english_annotation_ht{$english})
|
|
&& ($english =~ /\S/)) {
|
|
return ($maxlength, $english);
|
|
} else {
|
|
$maxlength--;
|
|
}
|
|
}
|
|
return (0,"");
|
|
}
|
|
|
|
sub select_reliable_entities {
|
|
local($caller, @pes) = @_;
|
|
|
|
foreach $i (0 .. $#pes) {
|
|
$pe = $pes[$i];
|
|
$surf = $pe->surf;
|
|
|
|
$pe->set("reliable",1);
|
|
}
|
|
return @pes;
|
|
}
|
|
|
|
sub negatives_p {
|
|
|
|
local($caller, $s1, $s2) = @_;
|
|
|
|
my $g_s1 = $util->regex_guard($s1);
|
|
my $g_s2 = $util->regex_guard($s2);
|
|
return 1 if $s1 =~ /^[iu]n$g_s2$/;
|
|
return 1 if $s1 =~ /^il$g_s2$/ && ($s2 =~ /^l/);
|
|
return 1 if $s1 =~ /^im$g_s2$/ && ($s2 =~ /^[mp]/);
|
|
|
|
return 1 if $s2 =~ /^[iu]n$g_s1$/;
|
|
return 1 if $s2 =~ /^il$g_s1$/ && ($s1 =~ /^l/);
|
|
return 1 if $s2 =~ /^im$g_s1$/ && ($s1 =~ /^[mp]/);
|
|
|
|
return 0;
|
|
}
|
|
|
|
sub present_participle_p {
|
|
local($caller, $pe) = @_;
|
|
|
|
my $aux_pe = $pe->child("aux");
|
|
return $caller->present_participle_p($aux_pe) if $aux_pe;
|
|
my $head_pe = $pe->child("head");
|
|
return $caller->present_participle_p($head_pe) if $head_pe;
|
|
return ($pe->synt =~ /^VBG/);
|
|
}
|
|
|
|
|
|
%engl_value_ht = (
|
|
"monday" => 1,
|
|
"tuesday" => 2,
|
|
"wednesday" => 3,
|
|
"thursday" => 4,
|
|
"friday" => 5,
|
|
"saturday" => 6,
|
|
"sunday" => 7,
|
|
|
|
"january" => 1,
|
|
"february" => 2,
|
|
"march" => 3,
|
|
"april" => 4,
|
|
"may" => 5,
|
|
"june" => 6,
|
|
"july" => 7,
|
|
"august" => 8,
|
|
"september" => 9,
|
|
"october" => 10,
|
|
"november" => 11,
|
|
"december" => 12,
|
|
|
|
"spring" => 1,
|
|
"summer" => 2,
|
|
"fall" => 3,
|
|
"autumn" => 3,
|
|
"winter" => 4,
|
|
|
|
"morning" => 1,
|
|
"noon" => 2,
|
|
"afternoon" => 3,
|
|
"evening" => 4,
|
|
"night" => 5,
|
|
|
|
"picosecond" => 1,
|
|
"nanosecond" => 2,
|
|
"microsecond" => 3,
|
|
"millisecond" => 4,
|
|
"second" => 5,
|
|
"minute" => 6,
|
|
"hour" => 7,
|
|
"day" => 8,
|
|
"week" => 9,
|
|
"fortnight" => 10,
|
|
"month" => 11,
|
|
"year" => 12,
|
|
"decade" => 13,
|
|
"century" => 14,
|
|
"millennium" => 15,
|
|
|
|
"nanometer" => 2,
|
|
"micrometer" => 3,
|
|
"millimeter" => 4,
|
|
"centimeter" => 5,
|
|
"decimeter" => 6,
|
|
"meter" => 7,
|
|
"kilometer" => 8,
|
|
"inch" => 11,
|
|
"foot" => 12,
|
|
"yard" => 13,
|
|
"mile" => 14,
|
|
"lightyear" => 20,
|
|
|
|
"microgram" => 2,
|
|
"milligram" => 3,
|
|
"gram" => 4,
|
|
"kilogram" => 5,
|
|
"ton" => 6,
|
|
"ounce" => 14,
|
|
);
|
|
|
|
sub engl_order_value {
|
|
local($this, $s) = @_;
|
|
|
|
return $value = $engl_value_ht{(lc $s)} || 0;
|
|
}
|
|
|
|
1;
|
|
|
|
|
