v3

Саян 29.11.2013 09:33

- упростил поведение в плане отглагольных прилагательных, вроде перестало глючить

#by father_gorry for Aktar CMS 
#after http://forum.dklab.ru/php/advises/HeuristicWithoutTheDictionaryExtractionOfARootFromRussianWord.html
#Thanks to Misha v.3 for corrections
#USAGE
#^stemmer:stem[$form:word;;3]
#OR $n[^stemmer::new[$form:word;;3]] ^n.4MBTS[] $n.prephrase $n.donephrase (table)

@USE
lib_aktar.p

@CLASS
stemmer_dummy

@auto[]
$noword[^table::create{f	t
?	 
.	 
,	 
!	 
:	 
&	 
^;	 
-	 
^taint[^#0A]	 }]


@as[regex] #this not only matches, but replaces caller RV
$prev_RV[$caller.RV]
$caller.RV[^prev_RV.match[$$regex][;]]
$result(^if($prev_RV ne $caller.RV){1;0})
#^msg[$regex = $result ${caller.start}$caller.RV]

@stem[phrase;ln;minlen]
^if(!def $ln){$ln[ru]}$language[$ln]
$initial_phrase[$phrase]
$minlen($minlen)

$prephrase[^ph2tab[$phrase]]

#$wache[^hashfile::open[/cache/stemmer/$language]]
$wache[^hafala:open[/cache/stemmer/$language]]
$donephrase[^table::create{word}]
^prephrase.menu{ 
	$w[^cache_word[$prephrase.piece]]
	^if(^w.length[] >= $minlen){^donephrase.append{$w}}
} 
#^wache.release[]

$result[^donephrase.menu{$donephrase.word}[ ]]

@ph2tab[phrase]
$phrase[^apply-taint[optimized-as-is][^phrase.replace[$noword]]]
$result[^phrase.split[ ;v]]

@new[phrase;ln;minlen][tmp]
$tmp[^stem[$phrase;$ln;$minlen]]

@4MBTS[]
$result[^donephrase.menu{$donephrase.word*}[ ]]

@cache_word[word][ccn]
$word[^prematch[^word.lower[]]]

^if(^word.length[] <= 3){
	$result[$word]
}{
	^if(def $wache.$word && 0){ 
		$result[$wache.$word]
	}{
		^if(!$stemmer_initialized){
			^init[]
			$stemmer_initialized(1)
		}
		$result[^stem_word[$word]]
		$wache.$word[$result]
	}
}

@prematch[w]
$result[$w]

@init[]

@stem_word[word]
$result[$word]

########################################################################
@CLASS
stemmer
#for different languages create different files and use them by necessity
@BASE
stemmer_dummy
@auto[]
$EREP[^table::create{f	t
ё	е}]

@prematch[word]
$result[^word.replace[$EREP]]

@new[phrase;ln;minlen][tmp]
$tmp[^stem[$phrase;$ln;$minlen]]

@init[] не выносить в @auto поскольку большинство запросов будет короче 3 букв и стеммер не нужен
$VOWEL[аеиоуыэюя]
$PERFECTIVEGROUND[^regex::create[((?:ив|ивши|ившись|ыв|ывши|ывшись)|((?<=[ая])(?:в|вши|вшись)))^$]]
$REFLEXIVE[^regex::create[(с[яь])^$]]
$ADJECTIVE[^regex::create[(?:ее|ие|ые|ое|ими|ыми|ей|ий|ый|ой|ем|им|ым|ом|его|ого|еых|ую|юю|ая|яя|ою|ею)^$]]
$PARTICIPLE[^regex::create[((?:ивш|ывш|ующ)|((?<=[ая])(?:ем|нн|вш|ющ|щ)))^$]]
$VERB[^regex::create[((?:ила|ыла|ена|ейте|уйте|ите|или|ыли|ей|уй|ил|ыл|им|ым|ены|ить|ыть|ишь|ую|ю)|((?<=[ая])(ла|на|ете|йте|ли|й|л|ем|н|ло|но|ет|ют|ны|ть|ешь|нно)))^$]]
$NOUN[^regex::create[(?:а|ев|ов|ие|ье|е|иями|ями|ям|ами|еи|ии|и|ией|ей|ой|ий|й|и|ы|ь|ию|ью|ю|ия|ья|я)^$]]
#$RVRE[^regex::create[^^(.*[аеиоуыэюя])(.*)^$]]
#todo: probably must extract only last RE
$RVRE[^regex::create[^^(.*?[аеиоуыэюя])(.*)^$]]
#what for this needed???
#$DERIVATIONAL[^regex::create[[^^аеиоуыэюя][аеиоуыэюя]+[^^аеиоуыэюя]+[аеиоуыэюя].*(?<=о)сть?^$]]
$DERIVATIONAL[^regex::create[(?:ост|аст|ист)^$]]
#$ADJNOUN[^regex::create[ость?^$]]

$SOFT[^regex::create[ь^$]]
$SUPERLATIVE[^regex::create[ейше?]]
$NN[^regex::create[нн^$]]

@stem_word[word][tmp]
$tmp[^word.match[$RVRE][]{$start[$match.1]$RV[$match.2]}]

^if(!def $RV){
	$result[$word]
;
	$result[^actual_stem_ru[$word;$RV;$start]]
}

@actual_stem_ru[word;RV;start]
^if(!^as[PERFECTIVEGROUND]){^as[REFLEXIVE]}
^if(^as[ADJECTIVE]){
	^as[PARTICIPLE]
}{
	^if(!^as[VERB]){^as[NOUN]}
} 
$RV[^RV.match[и^$][;]]
#see in @init
#^if(^as[DERIVATIONAL]){^as[ADJNOUN]}
^as[DERIVATIONAL]
^if(!^as[SOFT]){
	^as[SUPERLATIVE]
	$RV[^RV.match[$NN][;н]]
}
$result[${start}$RV]

---