TDPipe

#! /bin/bash -e

# ===
# This version of the pipeline tears early, to also protect Marmot.
# ===

# This is *one* way to use VRT tools to build a Finnish pipeline.
# Best run as array jobs in batch, see bin/gamarr for that.
# For convenience, consider a separate log dir for each such job.

# Assuming
# (1) VRT tools in bin/ (e.g., a symbolic link to a repo clone)
# (2) data/ as produced by bin/vrt-pack, q.v.
# (3) default field names
# fills data/ with siblings to *.vrf found there,
# in successive stages of analysis.

# Note! If there be quite inordinate "sentence" lengths, even marmot
# needs protection! and lookup/fillup (omorfi) may need protection
# against (long) words that look like multiple compounds (there is a
# tool to REDACT such). See bin/vrt-report-element-size for sentence
# length distribution, esp. max, and something may be written to
# report word lengths (bin/vrt-keep -n word | wc -L may help ISWIM).

case $# in
    1) ;;
    *)
	>&2 echo "$0: want 1 argument: got $#:"
	for arg in "$@"
	do
	    >&2 echo "argument: $arg"
	done
	exit 2
	;;
esac

tick () {
    now=$SECONDS
    add=$((now - TICK))
    printf -v FULL "%d:%02d:%02d" $((now/3600)) $((now/60%60)) $((now%60))
    printf -v TIME "%d:%02d:%02d" $((add/3600)) $((add/60%60)) $((add%60))
    date "+%F %T IN $TIME OF $FULL PAST $1"
    TICK=$now
}

case "$1" in
    data/a???/m???.vrf)

	# The A, B, C, D extensions are so that the lexicographic
	# order of the names is also their temporal order (and the
	# current notion of sibling extensions allows only letters).

	TICK=$SECONDS

	# tear/mend protect marmot/parser and long sentences from each other
	# (for a moderate sense of "long")
	# (and "sentence" can be most anything)

	bin/vrt-simple-tear -I Atear "$1"
	tick "tear"

	# tag : word -> marmot.in -> marmot.{pos,feat} -> lemma,pos,feat

	bin/vrt-tdp-alpha-lookup -I Atear/Blookup "$1.Atear"
	tick "lookup"

	bin/vrt-tdp-alpha-marmot -I Blookup/Cmarmot "$1.Blookup"
	tick "marmot"

	bin/vrt-tdp-alpha-fillup -I Cmarmot/Dfillup "$1.Cmarmot"
	tick "fillup"

	# parse : pos,feat -> id + head + rel

	bin/vrt-tdp-alpha-parse -I Dfillup/Eparse "$1.Dfillup"
	tick "parse"

	bin/vrt-conll09-mend -i "$1.Eparse"
	tick "mend"

	# Finish: id, head, rel => ref, dephead, deprel,
	# with ref nee id pre-fronted.

	bin/vrt-rename -I Eparse/Final \
		       -m id=ref,head=dephead,rel=deprel \
		       "$1.Eparse"

	bin/vrt-drop -i --dots "$1.Final"
	bin/vrt-keep -i -f word,ref --rest "$1.Final"

	tick "finish"

	;;

    *)
	>&2 echo "$0: want argument in data/a???/m???.vrf"
	>&2 echo "$0: got argument: $1"
	exit 2
	;;

esac