MRA-763 and others

package.json

@@ -14,7 +14,7 @@
 		"url": "[email protected]:natlibfi/melinda-marc-record-merge-reducers-js.git"
 	},
 	"license": "LGPL-3.0+",
-	"version": "2.1.0",
+	"version": "2.2.0-alpha.2",
 	"main": "./dist/index.js",
 	"engines": {
 		"node": ">=18"
@@ -39,10 +39,10 @@
 		"@natlibfi/marc-record": "^8.1.0",
 		"@natlibfi/marc-record-merge": "^7.0.2",
 		"@natlibfi/marc-record-validate": "^8.0.6",
-		"@natlibfi/marc-record-validators-melinda": "^10.16.0",
+		"@natlibfi/marc-record-validators-melinda": "^10.16.1",
 		"@natlibfi/melinda-commons": "^13.0.12",
 		"debug": "^4.3.4",
-		"isbn3": "^1.1.45",
+		"isbn3": "^1.1.46",
 		"normalize-diacritics": "^4.0.3"
 	},
 	"devDependencies": {
@@ -58,7 +58,7 @@
 		"babel-plugin-rewire": "^1.2.0",
 		"chai": "^4.4.1",
 		"cross-env": "^7.0.3",
-		"eslint": "^8.56.0",
+		"eslint": "^8.57.0",
 		"mocha": "^10.3.0",
 		"nodemon": "^3.0.3",
 		"nyc": "^15.1.0"

src/reducers/config.json

@@ -11,6 +11,42 @@
                 }
             },
 
+            {
+                "operation": "removeField",
+                "recordType": "source",
+                "comment": "MRA-763: remove many fields from a source prepublication if base record is good enough",
+                "fieldSpecification": {
+                    "tagPattern": "^((?!LOW|SID|00[135678]|035|040|042|500|506|540|594|776|884|946|974).)*$",
+                    "encodingLevel": ["8"]
+                },
+                "requireSourceField": {
+                    "tag": "500",
+                    "subfieldFilters": [{"code": "a", "valuePattern": "^(?:TARKISTETTU )?ENNAKKOTIETO"}]
+                },
+                "requireBaseField": {
+                    "tag": "...",
+                    "encodingLevel": [" ", "4", "5"]
+                }
+            },
+
+            {
+                "operation": "removeField",
+                "recordType": "source",
+                "comment": "MRA-763: remove many fields from a source prepublication if base record is good enough (Koneellisesti tuotettu tietue)",
+                "fieldSpecification": {
+                    "tagPattern": "^((?!LOW|SID|00[135678]|035|040|042|500|506|540|594|776|884|946|974).)*$",
+                    "encodingLevel": ["2", "8"]
+                },
+                "requireSourceField": {
+                    "tag": "500",
+                    "subfieldFilters": [{"code": "a", "valuePattern": "^Koneellisesti tuotettu tietue"}]
+                },
+                "requireBaseField": {
+                    "tagPattern": "...",
+                    "encodingLevel": [" ", "4", "5"]
+                }
+            },
+
             {
                 "operation": "removeField",
                 "recordType": "source",
@@ -25,6 +61,8 @@
                 }
             },
 
+
+
             {
                 "operation": "removeSubfield",
                 "comment": "Remove X00$d with old and deprecated (1) style value. (1) ... (99) are removed.",

src/reducers/controlSubfields.js

@@ -1,6 +1,6 @@
 import {MarcRecord} from '@natlibfi/marc-record';
 import createDebugLogger from 'debug';
-import {fieldHasSubfield, fieldToString, nvdebug, nvdebugSubfieldArray, subfieldIsRepeatable, subfieldToString, subfieldsAreIdentical} from './utils.js';
+import {fieldHasSubfield, fieldToString, nvdebug, nvdebugSubfieldArray, subfieldIsRepeatable, subfieldsAreIdentical} from './utils.js';
 
 //import {normalizeControlSubfieldValue} from './normalizeIdentifier';
 import {normalizeControlSubfieldValue} from '@natlibfi/marc-record-validators-melinda/dist/normalize-identifiers';
@@ -85,10 +85,10 @@
   const baseFieldSubfields9 = baseField.subfields.filter(sf => sf.code === '9');
   const sourceFieldSubfields9 = sourceField.subfields.filter(sf => sf.code === '9');
 
-  nvdebug('CHECK $9', debugDev);
+  //nvdebug('CHECK $9', debugDev);
   // There are no $9s. Skip:
   if (baseFieldSubfields9.length === 0 && sourceFieldSubfields9.length === 0) {
-    nvdebug(` No subfield $9 detected`, debugDev);
+    //nvdebug(` No subfield $9 detected`, debugDev);
     return true;
   }
 
@@ -102,12 +102,12 @@
     return false;
   }
 
-  nvdebug('CHECK $9 OK', debugDev);
+  //nvdebug('CHECK $9 OK', debugDev);
 
   return true;
 
   function subfieldHasKeepOrDrop(subfield) {
-    nvdebug(`Has <KEEP>? ${subfieldToString(subfield)}`, debugDev);
+    // nvdebug(`Has <KEEP>? ${subfieldToString(subfield)}`, debugDev);
     return subfield.code === '9' && (/(?:<KEEP>|<DROP>)/u).test(subfield.value);
   }
 
@@ -160,7 +160,7 @@
     return sourceSubfield.value === subfieldValue;
   }
 
   function keepOrDropPreventsMerge() {
     const keepOrDrop1 = baseFieldSubfields9.filter(sf => subfieldHasKeepOrDrop(sf));
     const keepOrDrop2 = sourceFieldSubfields9.filter(sf => subfieldHasKeepOrDrop(sf));
 

src/reducers/mergeSubfield.js

@@ -194,7 +194,7 @@ function preferHttpsOverHttp(candSubfield, relevantSubfields) {
 }
 
 function preferQualifierVersion(field, candSubfield, relevantSubfields) {
-  if (!fieldAllowsQualifierInOneOfTheSubfields(field, candSubfield) || !candSubfield.value.includes('(')) {
+  if (!fieldAllowsQualifierInOneOfTheSubfields(field, candSubfield)) { // || !candSubfield.value.includes('(')) {
     return false;
   }
 

src/reducers/preprocessor.js

@@ -14,6 +14,7 @@ import {default as fixSourceOfTerm} from '@natlibfi/marc-record-validators-melin
 import {default as modernize540} from '@natlibfi/marc-record-validators-melinda/dist/update-field-540';
 import {default as normalize505} from '@natlibfi/marc-record-validators-melinda/dist/field-505-separators';
 import {default as normalizeQualifyingInformation} from '@natlibfi/marc-record-validators-melinda/dist/normalize-qualifying-information';
+import {default as normalizeVariousSubfields} from '@natlibfi/marc-record-validators-melinda/dist/subfieldValueNormalizations';
 
 //const debug = createDebugLogger('@natlibfi/melinda-marc-record-merge-reducers:preprocessor');
 //const debugData = debug.extend('data');
@@ -46,6 +47,9 @@ export default (config = defaultConfig) => (base, source) => {
   normalizeQualifyingInformation().fix(base); // Modernize 015/020/024/028$q
   normalizeQualifyingInformation().fix(source);
 
+  normalizeVariousSubfields().fix(base); // Capitalize 130/240/243/600/.../830$l value etc
+  normalizeVariousSubfields().fix(source);
+
   normalize505().fix(base);
   normalize505().fix(source);
 

src/reducers/processFilter.js

@@ -164,12 +164,26 @@ function filterFieldsUsingFieldToString(fields, value) {
   });
 }
 
+function hasValidEncodingLevel(record, fieldSpecs) {
+  if (!fieldSpecs.encodingLevel) {
+    return true;
+  }
+  const recordEncodingLevel = getEncodingLevel(record);
+  //nvdebug(`ENC: ${recordEncodingLevel} in [${fieldSpecs.encodingLevel.join('')}]?`);
+  return fieldSpecs.encodingLevel.includes(recordEncodingLevel);
+}
+
 function getSpecifiedFieldsAndFilterThem(record, fieldSpecs) {
+  if (!hasValidEncodingLevel(record, fieldSpecs)) {
+    return [];
+  }
+
   const targetFields = getSpecifiedFields(record, fieldSpecs);
+  //nvdebug(`Got ${targetFields.length} fields. Filter them...`, debugDev);
   if (targetFields.length === 0) {
     return targetFields;
   }
-  //nvdebug(`Got ${targetFields.length} fields. Filter them...`, debugDev);
+
   const filteredFields1 = filterFieldsUsingSubfieldFilters(targetFields, fieldSpecs.subfieldFilters);
   //nvdebug(`${filteredFields1.length} field(s) remain after subfield filters...`, debugDev);
   const filteredFields2 = filterFieldsUsingFieldToString(filteredFields1, fieldSpecs.value);
@@ -218,7 +232,7 @@ function operationRemoveField(record, fieldSpecification) {
   if (deletableFields.length === 0) {
     return;
   }
-  nvdebug(`operationRemoveField got ${deletableFields.length} deletable field(s)`, debugDev);
+  //nvdebug(`operationRemoveField got ${deletableFields.length} deletable field(s)`, debugDev);
   deletableFields.forEach(field => {
     nvdebug(`  DELETE FIELD: ${fieldToString(field)}`, debugDev);
     record.removeField(field);
@@ -297,7 +311,7 @@ export function filterOperation(base, source, operation) {
     return;
   }
   const targetRecords = getTargetRecordsForOperation(base, source, operation);
-
+  //nvdebug(`filterOps: ${operation.comment ? operation.comment : 'NIMETÖN'}`);
   if (targetRecords.length === 0) {
     nvdebug('Failed to get the target record', debugDev);
     return;
@@ -327,6 +341,15 @@ export function filterOperation(base, source, operation) {
       //nvdebug(` Base field ${fieldToString(baseFields[0])}`, debugDev);
     }
 
+    if (operation.requireSourceField) {
+      const sourceFields = getSpecifiedFieldsAndFilterThem(source, operation.requireSourceField);
+      if (sourceFields.length === 0) {
+        //nvdebug(' Required source field not found!', debugDev);
+        return;
+      }
+      //nvdebug(` Base field ${fieldToString(baseFields[0])}`, debugDev);
+    }
+
     /*
     if (operation.requireOtherField) {
       // Does base === targetRecord work or do I need to clone this?
@@ -339,6 +362,7 @@ export function filterOperation(base, source, operation) {
   }
 
   function performActualOperation(targetRecord, operation, otherRecord) {
+    //nvdebug(' PERFORM OP');
     if (!operation.operation) {
       nvdebug('No operation defined', debugDev);
       return;

test-fixtures/reducers/index/MRA-412/base.json

@@ -0,0 +1,29 @@
+{
+  "leader": "01331cam a22003494i 4500",
+  "fields": [
+    { "tag": "800", "ind1": "1", "ind2": " ",
+    "subfields": [
+      { "code": "a", "value": "Jaakko, Petteri," },
+      { "code": "e", "value": "kirjoittaja." },
+      { "code": "t", "value": "Etsiväsarja ;" },
+      { "code": "v", "value": "14."}
+
+    ]
+  },
+    { "tag": "800", "ind1": "1", "ind2": " ",
+      "subfields": [
+        { "code": "a", "value": "Mäki, Susanna," },
+        { "code": "t", "value": "Simo Senjasen tapaukset ;" },
+        { "code": "v", "value": "[4] ."}
+      ]
+    },
+    { "tag": "800", "ind1": "1", "ind2": " ",
+    "subfields": [
+      { "code": "a", "value": "Golffari, Krisu," },
+      { "code": "t", "value": "Tarinoiden maa;" },
+      { "code": "v", "value": "2,"},
+      { "code": "l", "value": "espanja."}
+    ]
+  }
+  ]
+}

test-fixtures/reducers/index/MRA-412/merged.json

@@ -0,0 +1,29 @@
+{
+  "leader": "01331cam a22003494i 4500",
+  "fields": [
+    { "tag": "800", "ind1": "1", "ind2": " ",
+    "subfields": [
+      { "code": "a", "value": "Jaakko, Petteri," },
+      { "code": "e", "value": "kirjoittaja." },
+      { "code": "t", "value": "Etsiväsarja ;" },
+      { "code": "v", "value": "14."},
+      { "code": "l", "value": "Suomi."}
+    ]
+  },
+    { "tag": "800", "ind1": "1", "ind2": " ",
+      "subfields": [
+        { "code": "a", "value": "Mäki, Susanna," },
+        { "code": "t", "value": "Simo Senjasen tapaukset ;" },
+        { "code": "v", "value": "[4] ."}
+      ]
+    },
+    { "tag": "800", "ind1": "1", "ind2": " ",
+    "subfields": [
+      { "code": "a", "value": "Golffari, Krisu." },
+      { "code": "t", "value": "Tarinoiden maa ;" },
+      { "code": "v", "value": "2."},
+      { "code": "l", "value": "Espanja."}
+    ]
+  }
+  ]
+}

test-fixtures/reducers/index/MRA-412/metadata.json

@@ -0,0 +1,6 @@
+{
+  "description":"MRA-412 (all): field 800 title part issues",
+  "comment": "Sources are educated guesses.",
+  "comment #2": "$l-prepuncs: merged subfields trigger punc fix, $l capitalization is done later on",
+  "only": false
+}

test-fixtures/reducers/index/MRA-412/modifiedSource.json

@@ -0,0 +1,6 @@
+{
+  "leader": "01331cam a22003494i 4500",
+  "fields": [
+    { "tag": "001", "value": "123456" }
+  ]
+}

test-fixtures/reducers/index/MRA-412/source.json

@@ -0,0 +1,30 @@
+{
+  "leader": "01331cam a22003494i 4500",
+  "fields": [
+    { "tag": "001", "value": "123456" },
+    { "tag": "800", "ind1": "1", "ind2": " ",
+    "subfields": [
+      { "code": "a", "value": "Jaakko, Petteri," },
+      { "code": "e", "value": "kirjoittaja." },
+      { "code": "t", "value": "Etsiväsarja ;" },
+      { "code": "v", "value": "14,"},
+      { "code": "l", "value": "suomi." }
+
+    ]
+  },
+    { "tag": "800", "ind1": "1", "ind2": " ",
+      "subfields": [
+        { "code": "a", "value": "Mäki, Susanna," },
+        { "code": "t", "value": "Simo Senjasen tapaukset ;" },
+        { "code": "v", "value": "[4]."}
+      ]
+    },
+    { "tag": "800", "ind1": "1", "ind2": " ",
+    "subfields": [
+      { "code": "a", "value": "Golffari, Krisu," },
+      { "code": "t", "value": "Tarinoiden maa;" },
+      { "code": "v", "value": "2."}
+    ]
+  }
+  ]
+}

test-fixtures/reducers/index/MRA-763_clean_prepublication/base.json

@@ -0,0 +1,9 @@
+{
+  "leader": "01331cam a22003494i 4500",
+  "fields": [
+    { "tag": "001", "value": "123456" },
+    { "tag": "300", "ind1": " ", "ind2": " ", "subfields": [
+      { "code": "a", "value": "1 verkkoaineisto (7 h 36 min)" }
+    ]}
+  ]
+}

test-fixtures/reducers/index/MRA-763_clean_prepublication/merged.json

@@ -0,0 +1,15 @@
+{
+  "leader": "01331cam a22003494i 4500",
+  "fields": [
+    { "tag": "001", "value": "123456" },
+    { "tag": "300", "ind1": " ", "ind2": " ", "subfields": [
+      { "code": "a", "value": "1 verkkoaineisto (7 h 36 min)" }
+    ]},
+    { "tag": "500", "ind1": " ", "ind2": " ", "subfields": [ { "code": "a", "value": "Sen sijaan tämä kommentti mergeytyy." } ]},
+    { "tag": "SID", "ind1": " ", "ind2": " ", "subfields": [
+      { "code": "c", "value": "123" },
+      { "code": "b", "value": "owner" }
+    ]}
+
+  ]
+}

test-fixtures/reducers/index/MRA-763_clean_prepublication/metadata.json

@@ -0,0 +1,4 @@
+{
+  "description":"MRA-763: if the source record is a prepub, keep only certain tags",
+  "only": false
+}

test-fixtures/reducers/index/MRA-763_clean_prepublication/modifiedSource.json

@@ -0,0 +1,6 @@
+{
+  "leader": "01331cam a22003498i 4500",
+  "fields": [
+    { "tag": "001", "value": "123456" }
+  ]
+}

test-fixtures/reducers/index/MRA-763_clean_prepublication/source.json

@@ -0,0 +1,20 @@
+{
+  "leader": "01331cam a22003498i 4500",
+  "fields": [
+    { "tag": "001", "value": "123456" },
+    { "tag": "336", "ind1": " ", "ind2": " ", "subfields": [
+      { "code": "a", "value": "teksti" },
+      { "code": "b", "value": "txt" },
+      { "code": "2", "value": "rdamedia" }
+    ]},
+
+    { "tag": "500", "ind1": " ", "ind2": " ", "subfields": [ { "code": "a", "value": "TARKISTETTU ENNAKKOTIETO ei mergeydy!" } ]},
+    { "tag": "500", "ind1": " ", "ind2": " ", "subfields": [ { "code": "a", "value": "Sen sijaan tämä kommentti mergeytyy." } ]},
+    { "tag": "502", "ind1": " ", "ind2": " ", "subfields": [ { "code": "a", "value": "Väitöskirja." } ]},
+    { "tag": "LKR", "ind1": " ", "ind2": " ", "subfields": [ { "code": "a", "value": "sokeritoukka" } ]},
+    { "tag": "SID", "ind1": " ", "ind2": " ", "subfields": [
+      { "code": "c", "value": "123" },
+      { "code": "b", "value": "owner" }
+    ]}
+  ]
+}