Feature/merge works clean

librisworks/.gitignore

@@ -0,0 +1 @@
+/reports

librisworks/build.gradle

@@ -0,0 +1,37 @@
+apply plugin: 'application'
+apply plugin: 'groovy'
+
+sourceSets {
+    scripts {
+        groovy { srcDir 'scripts' }
+    }
+}
+
+repositories {
+    mavenCentral()
+}
+
+dependencies {
+    implementation project(':whelktool')
+    compileOnly "org.codehaus.groovy:groovy:${groovyVersion}"
+    compileOnly project(':whelk-core')
+    scriptsCompileOnly sourceSets.main.output
+    scriptsCompileOnly project(':whelk-core')
+}
+
+jar {
+    manifest {
+        attributes "Main-Class": "whelk.datatool.WhelkTool",
+                // log4j uses multi-release to ship different stack walking implementations for different java
+                // versions. Since we repackage everything as a fat jar, that jar must also be multi-release.
+                "Multi-Release": true
+    }
+
+    duplicatesStrategy = DuplicatesStrategy.EXCLUDE
+    from {
+        configurations.runtimeClasspath.collect {
+            it.isDirectory() ? it : project.zipTree(it).matching {
+            }
+        }
+    }
+}

librisworks/run.sh

@@ -0,0 +1,124 @@
+#!/bin/bash
+
+count_lines() {
+  if [ -f $1 ]; then
+    wc -l $1 | cut -d ' ' -f 1
+  else
+    echo 0
+  fi
+}
+
+if ! [[ "$1" =~ ^(local|dev|dev2|qa|stg|prod)$ ]]; then
+  echo "Missing or invalid environment"
+  exit 1
+fi
+
+ENV=$1
+ARGS="${@:2}"
+NUM_CLUSTERS=0
+
+JAR_FILE=build/libs/librisworks.jar
+
+SCRIPTS_DIR=scripts
+REPORT_DIR=reports/merge-works/$ENV-$(date +%Y%m%d)
+
+mkdir -p $REPORT_DIR/{clusters,normalizations,merged-works}
+
+CLUSTERS_DIR=$REPORT_DIR/clusters
+NORMALIZATIONS_DIR=$REPORT_DIR/normalizations
+
+FIND_CLUSTERS=$CLUSTERS_DIR/find-clusters
+ALL_CLUSTERS=$CLUSTERS_DIR/1-all.tsv
+MERGED_CLUSTERS=$CLUSTERS_DIR/2-merged.tsv
+TITLE_CLUSTERS=$CLUSTERS_DIR/3-title-clusters.tsv
+SWEDISH_FICTION=$CLUSTERS_DIR/4-swedish-fiction.tsv
+NO_ANONYMOUS_TRANSLATIONS=$CLUSTERS_DIR/5-no-anonymous-translations.tsv
+
+LANGUAGE_IN_TITLE=$NORMALIZATIONS_DIR/1-titles-with-language
+ELIB_DESIGNERS=$NORMALIZATIONS_DIR/2-elib-cover-designer
+CONTRIBUTION=$NORMALIZATIONS_DIR/3-contribution
+ROLES_TO_INSTANCE=$NORMALIZATIONS_DIR/4-roles-to-instance
+
+# Clustering step 1 TODO: run only on recently updated records after first run
+echo "Finding new clusters..."
+time java -Dxl.secret.properties=$HOME/secret.properties-$ENV -jar $JAR_FILE \
+  $ARGS --report $FIND_CLUSTERS $SCRIPTS_DIR/find-work-clusters.groovy >$ALL_CLUSTERS 2>/dev/null
+NUM_CLUSTERS=$(count_lines $ALL_CLUSTERS)
+echo "$NUM_CLUSTERS clusters found"
+if [ $NUM_CLUSTERS == 0 ]; then
+  exit 0
+fi
+
+# Clustering step 2
+echo
+echo "Merging clusters..."
+time java -Dxl.secret.properties=$HOME/secret.properties-$ENV -Dclusters=$ALL_CLUSTERS -jar $JAR_FILE \
+  $ARGS $SCRIPTS_DIR/merge-clusters.groovy >$MERGED_CLUSTERS 2>/dev/null
+NUM_CLUSTERS=$(count_lines $MERGED_CLUSTERS)
+echo "Merged into $NUM_CLUSTERS clusters"
+if [ $NUM_CLUSTERS == 0 ]; then
+  exit 0
+fi
+
+# Clustering step 3
+echo
+echo "Finding title clusters..."
+time java -Dxl.secret.properties=$HOME/secret.properties-$ENV -Dclusters=$MERGED_CLUSTERS -jar $JAR_FILE \
+  $ARGS $SCRIPTS_DIR/title-clusters.groovy >$TITLE_CLUSTERS 2>/dev/null
+NUM_CLUSTERS=$(count_lines $TITLE_CLUSTERS)
+echo "$NUM_CLUSTERS title clusters found"
+if [ $NUM_CLUSTERS == 0 ]; then
+  exit 0
+fi
+
+# Filter: Swedish fiction
+echo
+echo "Filtering on Swedish fiction..."
+time java -Dxl.secret.properties=$HOME/secret.properties-$ENV -Dclusters=$TITLE_CLUSTERS -jar $JAR_FILE \
+  $ARGS $SCRIPTS_DIR/swedish-fiction.groovy >$SWEDISH_FICTION 2>/dev/null
+NUM_CLUSTERS=$(count_lines $SWEDISH_FICTION)
+echo "Found $NUM_CLUSTERS title clusters with Swedish fiction"
+if [ $NUM_CLUSTERS == 0 ]; then
+  exit 0
+fi
+
+# Normalization
+echo
+echo "Removing language from work titles..."
+time java -Dxl.secret.properties=$HOME/secret.properties-$ENV -Dclusters=$SWEDISH_FICTION -jar $JAR_FILE \
+  $ARGS --report $LANGUAGE_IN_TITLE $SCRIPTS_DIR/language-in-work-title.groovy 2>/dev/null
+echo "$(count_lines $LANGUAGE_IN_TITLE/MODIFIED.txt) records affected, report in $LANGUAGE_IN_TITLE"
+
+echo
+echo "Specifying designer roles in Elib records..." # NOTE: Not dependent on clustering, can be run anytime after ContributionByRoleStep has been deployed.
+time java -Dxl.secret.properties=$HOME/secret.properties-$ENV -jar $JAR_FILE \
+  $ARGS --report $ELIB_DESIGNERS $SCRIPTS_DIR/elib-unspecified-contributor.groovy 2>/dev/null
+echo "$(count_lines $ELIB_DESIGNERS/MODIFIED.txt) records affected, report in $ELIB_DESIGNERS"
+
+echo
+echo "Normalizing contribution..."
+time java -Dxl.secret.properties=$HOME/secret.properties-$ENV -Dclusters=$SWEDISH_FICTION -jar $JAR_FILE \
+  $ARGS --report $CONTRIBUTION $SCRIPTS_DIR/normalize-contribution.groovy 2>/dev/null
+echo "$(count_lines $CONTRIBUTION/MODIFIED.txt) records affected, report in $CONTRIBUTION"
+
+echo
+echo "Moving roles to instance..."
+time java -Dxl.secret.properties=$HOME/secret.properties-$ENV -Dclusters=$SWEDISH_FICTION -jar $JAR_FILE \
+  $ARGS --report $ROLES_TO_INSTANCE $SCRIPTS_DIR/contributions-to-instance.groovy 2>/dev/null
+echo "$(count_lines $ROLES_TO_INSTANCE/MODIFIED.txt) records affected, report in $ROLES_TO_INSTANCE"
+
+# Filter: Drop anonymous translations
+echo "Filtering out anonymous translations..."
+time java -Dxl.secret.properties=$HOME/secret.properties-$ENV -Dclusters=$SWEDISH_FICTION -jar $JAR_FILE \
+  $ARGS $SCRIPTS_DIR/drop-anonymous-translations.groovy >$NO_ANONYMOUS_TRANSLATIONS 2>/dev/null
+NUM_CLUSTERS=$(count_lines $NO_ANONYMOUS_TRANSLATIONS)
+echo "$NUM_CLUSTERS clusters ready for merge"
+if [ $NUM_CLUSTERS == 0 ]; then
+  exit 0
+fi
+
+# Merge
+echo
+echo "Merging..."
+time java -Dxl.secret.properties=$HOME/secret.properties-$ENV -Dclusters=$NO_ANONYMOUS_TRANSLATIONS -jar $JAR_FILE \
+  $ARGS --report $REPORT_DIR/merged-works $SCRIPTS_DIR/merge-works.groovy 2>/dev/null

librisworks/scripts/contributions-to-instance.groovy

@@ -0,0 +1,96 @@
+import whelk.Whelk
+
+import static se.kb.libris.mergeworks.Util.Relator
+import static whelk.JsonLd.ID_KEY
+import static whelk.JsonLd.TYPE_KEY
+
+report = getReportWriter('report.tsv')
+
+def ids = new File(System.getProperty('clusters')).collect { it.split('\t').collect { it.trim() } }.flatten()
+
+def whelk = getWhelk()
+def instanceRolesByDomain = whelk.resourceCache.relators.findResults {
+    if (it.domain) {
+        def domain = whelk.jsonld.toTermKey(it.domain[ID_KEY])
+        if (whelk.jsonld.isSubClassOf(domain, 'Embodiment')) it.subMap([ID_KEY])
+    }
+}
+def instanceRoles = instanceRolesByDomain + [Relator.ILLUSTRATOR, Relator.AUTHOR_OF_INTRO, Relator.AUTHOR_OF_AFTERWORD].collect { [(ID_KEY): it.iri] }
+
+selectByIds(ids) { bib ->
+    Map instance = bib.graph[1]
+    def work = instance.instanceOf
+    def contribution = work?.contribution
+
+    if (!contribution) return
+
+    def ill = [(ID_KEY): Relator.ILLUSTRATOR.iri]
+
+    def modified = false
+
+    contribution.removeAll { c ->
+        if (isPrimaryContribution(c)) return false
+
+        def toInstance = asList(c.role).intersect(instanceRoles)
+        if (toInstance.contains(ill)) {
+            if (has9pu(c) || isPictureBook(work) || isComics(work, bib.whelk) || isStillImage(work)) {
+                toInstance.remove(ill)
+            }
+        }
+        if (toInstance) {
+            instance['contribution'] = asList(instance['contribution']) + c.clone().tap { it['role'] = toInstance }
+            c['role'] = asList(c.role) - toInstance
+            modified = true
+            report.println([bib.doc.shortId, toInstance.collect { it[ID_KEY].split('/').last() }].join('\t'))
+            incrementStats('moved to instance', toInstance)
+            return c.role.isEmpty()
+        }
+
+        return false
+    }
+
+    if (contribution.isEmpty()) {
+        work.remove('contribution')
+    }
+
+    if (modified) {
+        bib.scheduleSave()
+    }
+}
+
+boolean isPrimaryContribution(Map contribution) {
+    contribution[TYPE_KEY] == 'PrimaryContribution'
+}
+
+boolean has9pu(Map contribution) {
+    asList(contribution.role).contains([(ID_KEY): Relator.PRIMARY_RIGHTS_HOLDER.iri])
+}
+
+boolean isStillImage(Map work) {
+    asList(work.contentType).contains([(ID_KEY): 'https://id.kb.se/term/rda/StillImage'])
+}
+
+boolean isPictureBook(Map work) {
+    def picBookTerms = [
+            'https://id.kb.se/term/barngf/Bilderb%C3%B6cker',
+            'https://id.kb.se/term/barngf/Sm%C3%A5barnsbilderb%C3%B6cker'
+    ].collect { [(ID_KEY): it] }
+
+    return asList(work.genreForm).any { it in picBookTerms }
+}
+
+boolean isComics(Map work, Whelk whelk) {
+    def comicsTerms = [
+            'https://id.kb.se/term/saogf/Tecknade%20serier',
+            'https://id.kb.se/term/barngf/Tecknade%20serier',
+            'https://id.kb.se/term/gmgpc/swe/Tecknade%20serier',
+            'https://id.kb.se/marc/ComicOrGraphicNovel',
+            'https://id.kb.se/marc/ComicStrip'
+    ].collect { [(ID_KEY): it] }
+
+    return asList(work.genreForm).any {
+        it in comicsTerms
+                || it[ID_KEY] && whelk.relations.isImpliedBy('https://id.kb.se/term/saogf/Tecknade%20serier', it[ID_KEY])
+                || asList(work.classification).any { it.code?.startsWith('Hci') }
+    }
+}

librisworks/scripts/display-clusters.groovy

@@ -0,0 +1,23 @@
+import se.kb.libris.mergeworks.Doc
+import se.kb.libris.mergeworks.Html
+
+htmlReport = getReportWriter('clusters.html')
+
+htmlReport.println(Html.START)
+
+new File(System.getProperty('clusters')).splitEachLine(~/[\t ]+/) { cluster ->
+    List<Doc> docs = Collections.synchronizedList([])
+
+    selectByIds(cluster) {
+        it.getVersions()
+                .reverse()
+                .find { getAtPath(it.data, it.workIdPath) == null }
+                ?.with { docs.add(new Doc(getWhelk, it)) }
+    }
+
+    docs.each { it.addComparisonProps() }
+
+    htmlReport.println(Html.clusterTable(docs) + Html.HORIZONTAL_RULE)
+}
+
+htmlReport.println(Html.END)

librisworks/scripts/display-works.groovy

@@ -0,0 +1,55 @@
+import se.kb.libris.mergeworks.Doc
+import se.kb.libris.mergeworks.Html
+import se.kb.libris.mergeworks.WorkComparator
+
+import static se.kb.libris.mergeworks.Util.partition
+
+htmlReport = getReportWriter('works.html')
+
+htmlReport.println(Html.START)
+
+new File(System.getProperty('clusters')).splitEachLine(~/[\t ]+/) { cluster ->
+    List<Doc> docs = Collections.synchronizedList([])
+
+    selectByIds(cluster) {
+        it.getVersions()
+                .reverse()
+                .find { getAtPath(it.data, it.workIdPath) == null }
+                ?.with { docs.add(new Doc(getWhelk, it)) }
+    }
+
+    WorkComparator c = new WorkComparator(WorkComparator.allFields(docs))
+
+    def workClusters = workClusters(docs, c).findAll { it.size() > 1 }
+
+    workClusters.collect { [createNewWork(c.merge(it))] + it }
+            .each { htmlReport.println(Html.clusterTable(it) + Html.HORIZONTAL_RULE) }
+}
+
+htmlReport.println(Html.END)
+
+Collection<Collection<Doc>> workClusters(Collection<Doc> docs, WorkComparator c) {
+    docs.each { it.addComparisonProps() }
+
+    def workClusters = partition(docs, { Doc a, Doc b -> c.sameWork(a, b) })
+            .each { work -> work.each { doc -> doc.removeComparisonProps() } }
+
+    return workClusters
+}
+
+Doc createNewWork(Map workData) {
+    workData['@id'] = "TEMPID#it"
+    Map data = [
+            "@graph": [
+                    [
+                            "@id"       : "TEMPID",
+                            "@type"     : "Record",
+                            "mainEntity": ["@id": "TEMPID#it"],
+
+                    ],
+                    workData
+            ]
+    ]
+
+    return new Doc(create(data))
+}

librisworks/scripts/drop-anonymous-translations.groovy

@@ -0,0 +1,17 @@
+import se.kb.libris.mergeworks.Doc
+
+new File(System.getProperty('clusters')).splitEachLine(~/[\t ]+/) { cluster ->
+    List<Doc> docs = Collections.synchronizedList([])
+    selectByIds(cluster) {
+        docs.add(new Doc(it))
+    }
+
+    def filtered = docs.split { it.instanceData }
+            .with { local, linked ->
+                linked + local.findAll { Doc d -> !d.isAnonymousTranslation() }
+            }
+
+    if (filtered.size() > 1) {
+        println(filtered.collect { Doc d -> d.shortId() }.join('\t'))
+    }
+}