Fix f4xx (#182)

* Update dependencies
* Unify f506/856 evaluation regarding open access material
* Fix broken tests
* f490 generator now uses dc.relation.issn-l if dc.relation.issn does not exist or have valid value
* f490 generator now validates format of ISSN information (note: validity of ISSN is not tested at this point)
* f490 generator parses seemingly valid value from within a string that may contain any suffix or prefix

src/transform/convert/common/generate4xx.js

@@ -1,3 +1,5 @@
+import {isValidIssn, parseIssnFromString} from '../util';
+
 /**
  * Generates field 490 ($a, $v, $x) if subfields $a or $x can be generated.
  * Values creation is based on are found from dc.relation.ispartofseries,
@@ -47,7 +49,11 @@ export function generate490({getFieldValues}) {
     }
 
     function generateSubfieldX(hasSeriesNumber) {
-      return getFieldValues('dc.relation.issn')
+      const issnFieldValues = getFieldValues('dc.relation.issn').map(parseIssnFromString).filter(isValidIssn);
+      const issnLFieldValues = getFieldValues('dc.relation.issn-l').map(parseIssnFromString).filter(isValidIssn);
+
+      const issnValues = issnFieldValues.length > 0 ? issnFieldValues : issnLFieldValues;
+      return issnValues
         .reduceRight((acc, value) => {
           if (acc.length === 0) {
             return acc.concat({code: 'x', value: hasSeriesNumber ? `${value} ;` : value});

src/transform/convert/common/generate5xx.js

@@ -1,4 +1,4 @@
-import {hasLevels, isDissertation, extractFinnishTerm} from '../util';
+import {hasLevels, isDissertation, extractFinnishTerm, isOpenAccess} from '../util';
 
 /**
  * Generates field 500 ($a) based on dc.description, dc.description.notification and dc.type.ontasot
@@ -128,12 +128,9 @@ export function generate506({getFieldValues}) {
   return accessLevelFields.concat(accessRightsFields);
 
   function generateAccessLevelFields() {
-    const accessLevelFields = getFieldValues('dc.rights.accesslevel');
-    const isOpenAccess = accessLevelFields.length === 0 || accessLevelFields
-      .filter(value => value === 'openAccess')
-      .length > 0;
+    const openAccess = isOpenAccess({getFieldValues});
 
-    return isOpenAccess ? [
+    return openAccess ? [
       {
         tag: '506',
         ind1: '0',

src/transform/convert/common/generate8xx.js

@@ -3,7 +3,7 @@ import {createHash} from 'crypto';
 import {clone} from '@natlibfi/melinda-commons';
 import {MarcRecord} from '@natlibfi/marc-record';
 
-import {isValidLink} from '../util';
+import {isOpenAccess, isValidLink} from '../util';
 import {sourceConfig} from '../../../config';
 
 /**
@@ -21,10 +21,10 @@ export function generate856({getFieldValues}) {
   return publicAccessFields.concat(otherUrnFields);
 
   function generatePublicAccessFields() {
-    const accessLevels = getFieldValues('dc.rights.accesslevel');
+    const openAccess = isOpenAccess({getFieldValues});
 
     // NB: not having access level fields generates f856 public access field
-    if (accessLevels.length === 0 || accessLevels.find(v => v === 'openAccess')) {
+    if (openAccess) {
       const subfields = generateSubfields();
       return subfields.length > 0 ? [{tag: '856', ind1: '4', ind2: '0', subfields}] : [];
     }

src/transform/convert/util/index.js

@@ -228,3 +228,34 @@ export function getAllValuesInContext(context, ...path) {
     return [];
   }
 }
+
+export function isOpenAccess({getFieldValues}) {
+  const dcAccessLevelFields = getFieldValues('dc.rights.accesslevel');
+  const dcAccessRightsFields = getFieldValues('dc.rights.accessrights');
+
+  const accessFields = [...dcAccessLevelFields, ...dcAccessRightsFields];
+
+  return accessFields.length === 0 || dcAccessLevelFields.includes('openAccess');
+}
+
+// This is lax validation -- if stricter is required it will be implemented at later stage and to common package
+export function isValidIssn(issn) {
+  if (typeof issn !== 'string' || issn.length < 8) {
+    return false;
+  }
+
+  const issnRegexLax = /^\d{4}-\d{3}[0-9xX]{1}$/u;
+  return issnRegexLax.test(issn);
+}
+
+export function parseIssnFromString(issnString) {
+  const issnRegex = /\d{4}-\d{3}[0-9xX]{1}/u;
+  const containsIssn = issnRegex.test(issnString);
+
+  if (!containsIssn) {
+    return null;
+  }
+
+  const [result] = issnString.match(issnRegex);
+  return result;
+}

test-fixtures/transform/convert/common/generate4xx/generate490/06/input.json

@@ -0,0 +1,10 @@
+[
+  {
+    "$": {
+      "schema": "dc",
+      "element": "relation",
+      "qualifier": "issn-l",
+      "value": "1459-3467"
+    }
+  }
+]

test-fixtures/transform/convert/common/generate4xx/generate490/06/metadata.json

@@ -0,0 +1,4 @@
+{
+  "description": "Value may be taken from dc.relation.issn-l if dc.relation.issn cannot be found",
+  "only": false
+}

test-fixtures/transform/convert/common/generate4xx/generate490/06/output.json

@@ -0,0 +1,8 @@
+[
+  {
+    "tag": "490", "ind1": "0", "ind2": " ",
+    "subfields": [
+      {"code": "x", "value": "1459-3467"}
+    ]
+  }
+]

test-fixtures/transform/convert/common/generate4xx/generate490/07/input.json

@@ -0,0 +1,18 @@
+[
+  {
+    "$": {
+      "schema": "dc",
+      "element": "relation",
+      "qualifier": "issn-l",
+      "value": "1234-4321"
+    }
+  },
+  {
+    "$": {
+      "schema": "dc",
+      "element": "relation",
+      "qualifier": "issn",
+      "value": "1459-3467"
+    }
+  }
+]

test-fixtures/transform/convert/common/generate4xx/generate490/07/metadata.json

@@ -0,0 +1,4 @@
+{
+  "description": "Value may not be taken from dc.relation.issn-l if dc.relation.issn can be found",
+  "only": false
+}

test-fixtures/transform/convert/common/generate4xx/generate490/07/output.json

@@ -0,0 +1,8 @@
+[
+  {
+    "tag": "490", "ind1": "0", "ind2": " ",
+    "subfields": [
+      {"code": "x", "value": "1459-3467"}
+    ]
+  }
+]

test-fixtures/transform/convert/common/generate4xx/generate490/08/input.json

@@ -0,0 +1,10 @@
+[
+  {
+    "$": {
+      "schema": "dc",
+      "element": "relation",
+      "qualifier": "issn",
+      "value": "totally-not-valid-issn"
+    }
+  }
+]

test-fixtures/transform/convert/common/generate4xx/generate490/08/metadata.json

@@ -0,0 +1,4 @@
+{
+  "description": "Won't use value from dc.relation.issn if it's not valid ISSN value (lax validation)",
+  "only": false
+}

test-fixtures/transform/convert/common/generate4xx/generate490/08/output.json

@@ -0,0 +1 @@
+[]

test-fixtures/transform/convert/common/generate4xx/generate490/09/input.json

@@ -0,0 +1,10 @@
+[
+  {
+    "$": {
+      "schema": "dc",
+      "element": "relation",
+      "qualifier": "issn-l",
+      "value": "totally-not-valid-issn"
+    }
+  }
+]

test-fixtures/transform/convert/common/generate4xx/generate490/09/metadata.json

@@ -0,0 +1,4 @@
+{
+  "description": "Won't use value from dc.relation.issn-l if it's not valid ISSN value (lax validation)",
+  "only": false
+}

test-fixtures/transform/convert/common/generate4xx/generate490/09/output.json

@@ -0,0 +1 @@
+[]

test-fixtures/transform/convert/common/generate4xx/generate490/10/input.json

@@ -0,0 +1,18 @@
+[
+  {
+    "$": {
+      "schema": "dc",
+      "element": "relation",
+      "qualifier": "issn-l",
+      "value": "1459-3467"
+    }
+  },
+  {
+    "$": {
+      "schema": "dc",
+      "element": "relation",
+      "qualifier": "issn",
+      "value": "totally-not-valid"
+    }
+  }
+]

test-fixtures/transform/convert/common/generate4xx/generate490/10/metadata.json

@@ -0,0 +1,4 @@
+{
+  "description": "Value may be taken from dc.relation.issn-l if dc.relation.issn can be found but is invalid",
+  "only": false
+}

test-fixtures/transform/convert/common/generate4xx/generate490/10/output.json

@@ -0,0 +1,8 @@
+[
+  {
+    "tag": "490", "ind1": "0", "ind2": " ",
+    "subfields": [
+      {"code": "x", "value": "1459-3467"}
+    ]
+  }
+]

test-fixtures/transform/convert/common/generate4xx/generate490/11/input.json

@@ -0,0 +1,18 @@
+[
+  {
+    "$": {
+      "schema": "dc",
+      "element": "relation",
+      "qualifier": "issn-l",
+      "value": "even-more-invalid"
+    }
+  },
+  {
+    "$": {
+      "schema": "dc",
+      "element": "relation",
+      "qualifier": "issn",
+      "value": "totally-not-valid"
+    }
+  }
+]

test-fixtures/transform/convert/common/generate4xx/generate490/11/metadata.json

@@ -0,0 +1,4 @@
+{
+  "description": "If both dc.relation.issn-l and dc.relation.issn can be found but are invalid, field is not generated",
+  "only": false
+}

test-fixtures/transform/convert/common/generate4xx/generate490/11/output.json

@@ -0,0 +1 @@
+[]

test-fixtures/transform/convert/common/generate4xx/generate490/12/input.json

@@ -0,0 +1,10 @@
+[
+  {
+    "$": {
+      "schema": "dc",
+      "element": "relation",
+      "qualifier": "issn",
+      "value": "ISSN 1459-3467 NSSI"
+    }
+  }
+]

test-fixtures/transform/convert/common/generate4xx/generate490/12/metadata.json

@@ -0,0 +1,4 @@
+{
+  "description": "If ISSN is contained within the value, it is correctly parsed from there",
+  "only": false
+}

test-fixtures/transform/convert/common/generate4xx/generate490/12/output.json

@@ -0,0 +1,8 @@
+[
+  {
+    "tag": "490", "ind1": "0", "ind2": " ",
+    "subfields": [
+      {"code": "x", "value": "1459-3467"}
+    ]
+  }
+]

test-fixtures/transform/convert/common/generate4xx/generate490/13/input.json

@@ -0,0 +1,10 @@
+[
+  {
+    "$": {
+      "schema": "dc",
+      "element": "relation",
+      "qualifier": "issn",
+      "value": "ISSN 1459-3467 NSSI 1234-4321"
+    }
+  }
+]

test-fixtures/transform/convert/common/generate4xx/generate490/13/metadata.json

@@ -0,0 +1,4 @@
+{
+  "description": "If multiple ISSN is contained within one value, only first one is parsed and used for field generation",
+  "only": false
+}

test-fixtures/transform/convert/common/generate4xx/generate490/13/output.json

@@ -0,0 +1,8 @@
+[
+  {
+    "tag": "490", "ind1": "0", "ind2": " ",
+    "subfields": [
+      {"code": "x", "value": "1459-3467"}
+    ]
+  }
+]

test-fixtures/transform/convert/common/generate5xx/generate506/03/metadata.json

@@ -1,4 +1,4 @@
 {
-  "description": "Empty input produces empty output",
+  "description": "Empty input produces open access field because lack of any dc.rights.accesslevel fields",
   "only": false
 }

test-fixtures/transform/convert/common/generate5xx/generate506/03/output.json

@@ -1 +1,11 @@
-[]
+[
+  {
+    "tag": "506", "ind1": "0", "ind2": "",
+    "subfields": [
+      {"code": "a", "value": "Aineisto on vapaasti saatavissa."},
+      {"code": "f", "value": "Unrestricted online access"},
+      {"code": "2", "value": "star"},
+      {"code": "9", "value": "FENNI<KEEP>"}
+    ]
+  }
+]