Migrar exportação Crossref do schema 4.4.0 para 5.5.0

articlemeta/export_crossref.py

@@ -27,9 +27,13 @@ def transform(self, data):
         }
 
         el = ET.Element('doi_batch', nsmap=nsmap)
-        el.set('version', '4.4.0')
-        el.set('xmlns', 'http://www.crossref.org/schema/4.4.0')
-        el.set('{http://www.w3.org/2001/XMLSchema-instance}schemaLocation', 'http://www.crossref.org/schema/4.4.0 http://www.crossref.org/schemas/crossref4.4.0.xsd')
+        el.set('version', '5.5.0')
+        el.set('xmlns', 'http://www.crossref.org/schema/5.5.0')
+        el.set(
+            '{http://www.w3.org/2001/XMLSchema-instance}schemaLocation',
+            'http://www.crossref.org/schema/5.5.0 '
+            'https://data.crossref.org/schemas/crossref5.5.0.xsd'
+        )
 
         return data, el
 
@@ -340,7 +344,6 @@ def _create_journal_article(language=None):
         if language:
             el.set('language', language)
         el.set('publication_type', 'full_text')
-        el.set('reference_distribution_opts', 'any')
         return el
 
     def transform(self, data):
@@ -449,6 +452,30 @@ def precond(data):
         if not raw.authors:
             raise plumber.UnmetPrecondition()
 
+    @staticmethod
+    def _create_institution(affiliation):
+        institution_name = affiliation.get('institution')
+        if not institution_name or not institution_name.strip():
+            return None
+
+        institution = ET.Element('institution')
+
+        name = ET.Element('institution_name')
+        name.text = institution_name
+        institution.append(name)
+
+        place_parts = [
+            affiliation.get(field)
+            for field in ('city', 'state', 'country')
+            if affiliation.get(field) and affiliation.get(field).strip()
+        ]
+        if place_parts:
+            place = ET.Element('institution_place')
+            place.text = ', '.join(place_parts)
+            institution.append(place)
+
+        return institution
+
     @plumber.precondition(precond)
     def transform(self, data):
         """
@@ -494,29 +521,20 @@ def transform(self, data):
             author_index = [i.upper() for i in authors.get('xref', []) or []]
 
             if raw.affiliations:
-                affs_list = []
+                affiliations = ET.Element('affiliations')
                 for aff in raw.affiliations:
-                    affiliation = ET.Element('affiliation')
-                    if 'index' not in aff:
+                    affiliation_index = aff.get('index')
+                    if not affiliation_index:
+                        continue
+                    if affiliation_index.upper() not in author_index:
                         continue
-                    if aff['index'].upper() in author_index:
-                        aff_list = []
-                        if 'institution' in aff:
-                            aff_list.append(aff['institution'])
-                        if 'addr_line' in aff:
-                            aff_list.append(aff['addr_line'])
-                        if 'country' in aff:
-                            aff_list.append(aff['country'])
-
-                        aff_info = ',  '.join(aff_list)
-                        if len(aff_info.strip()) == 0:
-                            continue
-                        affs_list.append(aff_info)
-
-                affs = '; '.join(affs_list)
-                if len(affs) > 0:
-                    affiliation.text = affs
-                    author.append(affiliation)
+
+                    institution = self._create_institution(aff)
+                    if institution is not None:
+                        affiliations.append(institution)
+
+                if len(affiliations):
+                    author.append(affiliations)
 
             if 'orcid' in authors and authors['orcid']:
                 orcid = ET.Element('ORCID')

setup.py

@@ -21,7 +21,13 @@
     'crossrefapi>=1.3',
     ]
 
-test_requires = ['mocker', 'nose>=1.0', 'coverage', 'mongomock']
+test_requires = [
+    'mocker',
+    'nose>=1.0',
+    'coverage',
+    'mongomock',
+    'xmlschema',
+]
 
 setup(
     name="articlemeta",

tests/test_export_crossref.py

@@ -2,16 +2,39 @@
 import unittest
 import json
 import os
-import io
-from unittest.mock import patch, PropertyMock
+from unittest.mock import Mock, patch, PropertyMock
 
 from lxml import etree as ET
+import xmlschema
 
 from articlemeta import export_crossref
 from articlemeta import export
 from articlemeta.export import CustomArticle as Article
 
 
+SCHEMA_DIR = os.path.join(
+    os.path.dirname(__file__), 'xsd', 'scielo_crossref')
+CROSSREF_SCHEMA_PATH = os.path.join(SCHEMA_DIR, 'crossref5.5.0.xsd')
+CROSSREF_SCHEMA_LOCATIONS = [
+    (
+        'http://www.ncbi.nlm.nih.gov/JATS1',
+        'https://data.crossref.org/schemas/'
+        'JATS-journalpublishing1-3d2-mathml3.xsd'
+    ),
+]
+_CROSSREF_SCHEMA = None
+
+
+def get_crossref_schema():
+    global _CROSSREF_SCHEMA
+    if _CROSSREF_SCHEMA is None:
+        _CROSSREF_SCHEMA = xmlschema.XMLSchema11(
+            CROSSREF_SCHEMA_PATH,
+            locations=CROSSREF_SCHEMA_LOCATIONS,
+        )
+    return _CROSSREF_SCHEMA
+
+
 def _get_article(data=None):
     article_json = {
         "fulltexts": {
@@ -251,6 +274,19 @@ def test_doi_batch_element(self):
         raw, xml = xmlcrossref.transform(data)
 
         self.assertEqual('doi_batch', xml.tag)
+        self.assertEqual('5.5.0', xml.get('version'))
+        self.assertEqual(
+            'http://www.crossref.org/schema/5.5.0',
+            xml.get('xmlns')
+        )
+        self.assertEqual(
+            'http://www.crossref.org/schema/5.5.0 '
+            'https://data.crossref.org/schemas/crossref5.5.0.xsd',
+            xml.get(
+                '{http://www.w3.org/2001/XMLSchema-instance}'
+                'schemaLocation'
+            )
+        )
 
     def test_doi_batch_id_element(self):
 
@@ -514,7 +550,7 @@ def test_journal_article_element(self):
         xmlcrossref = export_crossref.XMLJournalArticlePipe()
         raw, xml = xmlcrossref.transform(data)
 
-        self.assertEqual(b'<doi_batch><body><journal><journal_article language="pt" publication_type="full_text" reference_distribution_opts="any"/></journal></body></doi_batch>', ET.tostring(xml))
+        self.assertEqual(b'<doi_batch><body><journal><journal_article language="pt" publication_type="full_text"/></journal></body></doi_batch>', ET.tostring(xml))
 
     def test_journal_article_element_without_doi_and_lang(self):
         xmlcrossref = ET.Element('doi_batch')
@@ -535,8 +571,6 @@ def test_journal_article_element_without_doi_and_lang(self):
         self.assertEqual(1, len(journal_articles))
         self.assertEqual('pt', journal_articles[0].get('language'))
         self.assertEqual('full_text', journal_articles[0].get('publication_type'))
-        self.assertEqual(
-            'any', journal_articles[0].get('reference_distribution_opts'))
 
     def test_article_titles_element(self):
 
@@ -633,7 +667,89 @@ def test_article_contributors_element(self):
         xmlcrossref = export_crossref.XMLArticleContributorsPipe()
         raw, xml = xmlcrossref.transform(data)
 
-        self.assertEqual(b'<doi_batch><body><journal><journal_article publication_type="full_text"><contributors><person_name contributor_role="editor" sequence="first"><given_name>Mariangela Leal</given_name><surname>Cherchiglia</surname><affiliation>Universidade Federal de Minas Gerais,  BRAZIL</affiliation></person_name><person_name contributor_role="author" sequence="additional"><given_name>Elaine Leandro</given_name><surname>Machado</surname><affiliation>Universidade Federal de Minas Gerais,  BRAZIL</affiliation></person_name><person_name contributor_role="translator" sequence="additional"><given_name>Daniele Ara&#250;jo Campo</given_name><surname>Szuster</surname><affiliation>Universidade Federal de Minas Gerais,  BRAZIL</affiliation></person_name><person_name contributor_role="author" sequence="additional"><given_name>Eli Iola Gurgel</given_name><surname>Andrade</surname><affiliation>Universidade Federal de Minas Gerais,  BRAZIL</affiliation></person_name><person_name contributor_role="author" sequence="additional"><given_name>Francisco de Assis</given_name><surname>Ac&#250;rcio</surname><affiliation>Universidade Federal de Minas Gerais,  BRAZIL</affiliation></person_name><person_name contributor_role="author" sequence="additional"><given_name>Waleska Teixeira</given_name><surname>Caiaffa</surname><affiliation>Universidade Federal de Minas Gerais,  BRAZIL</affiliation></person_name><person_name contributor_role="author" sequence="additional"><given_name>Ricardo</given_name><surname>Sesso</surname><affiliation>Universidade Federal de S&#227;o Paulo,  BRAZIL</affiliation></person_name><person_name contributor_role="author" sequence="additional"><given_name>Augusto A</given_name><surname>Guerra Junior</surname><affiliation>Universidade Federal de Minas Gerais,  BRAZIL; Universidade Federal de S&#227;o Paulo,  BRAZIL</affiliation></person_name><person_name contributor_role="author" sequence="additional"><given_name>Odilon Vanni de</given_name><surname>Queiroz</surname><affiliation>Universidade Federal de Minas Gerais,  BRAZIL</affiliation></person_name><person_name contributor_role="author" sequence="additional"><given_name>Isabel Cristina</given_name><surname>Gomes</surname><affiliation>Universidade Federal de Minas Gerais,  BRAZIL</affiliation></person_name></contributors></journal_article></journal></body></doi_batch>', ET.tostring(xml))
+        persons = xml.findall('.//contributors/person_name')
+        self.assertEqual(10, len(persons))
+        self.assertEqual(
+            ['editor', 'author', 'translator'] + ['author'] * 7,
+            [person.get('contributor_role') for person in persons]
+        )
+        self.assertEqual([], xml.findall('.//affiliation'))
+
+        first_institution = persons[0].find(
+            './affiliations/institution')
+        self.assertEqual(
+            'Universidade Federal de Minas Gerais',
+            first_institution.findtext('institution_name')
+        )
+        self.assertEqual(
+            'Belo Horizonte, MG, BRAZIL',
+            first_institution.findtext('institution_place')
+        )
+
+        multiple_institutions = persons[7].findall(
+            './affiliations/institution')
+        self.assertEqual(2, len(multiple_institutions))
+        self.assertEqual(
+            [
+                'Universidade Federal de Minas Gerais',
+                'Universidade Federal de São Paulo',
+            ],
+            [
+                institution.findtext('institution_name')
+                for institution in multiple_institutions
+            ]
+        )
+
+    def test_article_affiliations_skip_invalid_entries_and_precede_orcid(self):
+        raw = Mock()
+        raw.authors = [
+            {
+                'given_names': 'Jane',
+                'surname': 'Doe',
+                'xref': ['A1', 'A2', 'A3'],
+                'orcid': '0000-0002-1825-0097',
+            }
+        ]
+        raw.affiliations = [
+            {
+                'index': 'A1',
+                'institution': 'Example University',
+                'city': 'Example City',
+                'state': 'EX',
+                'country': 'Example Country',
+            },
+            {
+                'index': 'A2',
+                'institution': '   ',
+                'city': 'Ignored City',
+            },
+            {
+                'index': 'A3',
+                'institution': 'Name Only Institute',
+                'city': ' ',
+                'country': '',
+            },
+        ]
+
+        xml = ET.fromstring(
+            '<doi_batch><body><journal><journal_article/>'
+            '</journal></body></doi_batch>'
+        )
+        pipe = export_crossref.XMLArticleContributorsPipe()
+        _, xml = pipe.transform([raw, xml])
+
+        person = xml.find('.//person_name')
+        self.assertEqual(
+            ['given_name', 'surname', 'affiliations', 'ORCID'],
+            [child.tag for child in person]
+        )
+        institutions = person.findall('./affiliations/institution')
+        self.assertEqual(2, len(institutions))
+        self.assertEqual(
+            'Example City, EX, Example Country',
+            institutions[0].findtext('institution_place')
+        )
+        self.assertIsNone(institutions[1].find('institution_place'))
 
     def test_article_publication_date_element(self):
 
@@ -812,17 +928,10 @@ def test_xmlclose_pipe(self):
     def test_validating_against_schema(self):
 
         xml = export.Export(self._raw_json).pipeline_crossref()
+        schema = get_crossref_schema()
 
-        xmlio = ET.parse(io.BytesIO(xml))
-
-        fp = open(os.path.dirname(__file__)+'/xsd/scielo_crossref/crossref4.4.0.xsd')
-        schema_root = ET.parse(fp)
-        schema = ET.XMLSchema(schema_root)
-        fp.close()
-
-        schema.assertValid(xmlio)
-        self.assertTrue(schema.validate(xmlio))
-        self.assertEqual(None, schema.assertValid(xmlio))
+        self.assertTrue(schema.is_valid(xml))
+        self.assertEqual(None, schema.validate(xml))
 
     def test_related_articles_validating_against_schema(self):
         related_documents = [
@@ -844,16 +953,10 @@ def test_related_articles_validating_against_schema(self):
                 return_value=related_documents):
             xml = export.Export(self._raw_json).pipeline_crossref()
 
-        xmlio = ET.parse(io.BytesIO(xml))
-        schema_path = (
-            os.path.dirname(__file__) +
-            '/xsd/scielo_crossref/crossref4.4.0.xsd'
-        )
-        with open(schema_path) as fp:
-            schema = ET.XMLSchema(ET.parse(fp))
-
-        schema.assertValid(xmlio)
-        relations = xmlio.findall(
+        schema = get_crossref_schema()
+        schema.validate(xml)
+        xmlroot = ET.fromstring(xml)
+        relations = xmlroot.findall(
             './/{http://www.crossref.org/relations.xsd}'
             'inter_work_relation'
         )
@@ -2738,14 +2841,14 @@ def setUp(self):
         ET.register_namespace('fr', "http://www.crossref.org/fundref.xsd")
 
         self.xmlcrossref =  ET.Element(
-            '{http://www.crossref.org/schema/4.4.0}doi_batch',
+            '{http://www.crossref.org/schema/5.5.0}doi_batch',
             nsmap=namespace_map,
             attrib={
                 '{http://www.w3.org/2001/XMLSchema-instance}schemaLocation': (
-                    "http://www.crossref.org/schema/4.4.0 "
-                    "http://www.crossref.org/schemas/crossref4.4.0.xsd"
+                    "http://www.crossref.org/schema/5.5.0 "
+                    "https://data.crossref.org/schemas/crossref5.5.0.xsd"
                 ),
-                'version': '4.4.0'
+                'version': '5.5.0'
             }
         )
         journal = ET.Element('journal')

tests/xsd/scielo_crossref/AccessIndicators.xsd

@@ -1,27 +1,24 @@
 <?xml version="1.0" encoding="UTF-8"?>
 <xsd:schema xmlns:xsd="http://www.w3.org/2001/XMLSchema"
     targetNamespace="http://www.crossref.org/AccessIndicators.xsd"
-    xmlns="http://www.crossref.org/AccessIndicators.xsd">
+    xmlns="http://www.crossref.org/AccessIndicators.xsd" version="1.2">
 
     <xsd:annotation>
-        <xsd:documentation> Version: 1.1 This is CrossRef's schema for defining the applicable
-            licenses for a given item. This schema was available and in use prior to the completion
-            of the NISO working group Access and License Indicators
-            (http://www.niso.org/publications/rp/rp-22-2015). That effort produced a schema
-            (http://www.niso.org/schemas/ali/1.0/ali.xsd) that extended the CrossRef definition but
-            at the same time omitted necessary CrossRef features. This schema will continue as the
-            basis for CrossRef metadata deposits, but will incorporate the NISO work where possible.
-            Change history: 2/23/15 CSK added Niso free_to_read element 
-                            4/21/15 CSK added start and end attributes to the free-to-read element as in the Niso ALI schema
-                                        but will make both attributes optional.
+        <xsd:documentation> 
+
+Version 1.2: updated to include stm-asf value for @applies_to attribute 
+
+Version 1.1 notes:
+
+This is CrossRef's schema for defining the applicable licenses for a given item. This schema was available and in use prior to the completion  of the NISO working group Access and License Indicators (http://www.niso.org/publications/rp/rp-22-2015). That effort produced a schema (http://www.niso.org/schemas/ali/1.0/ali.xsd) that extended the CrossRef definition but at the same time omitted necessary CrossRef features. This schema will continue as the basis for CrossRef metadata deposits, but will incorporate the NISO work where possible.
+    Change history: 2/23/15 CSK added Niso free_to_read element 
+    4/21/15 CSK added start and end attributes to the free-to-read element as in the Niso ALI schema but will make both attributes optional.
         </xsd:documentation>
     </xsd:annotation>
 
     <xsd:element name="program">
         <xsd:annotation>
-            <xsd:documentation>Accommodates deposit of license metadata. The license_ref value will
-                be a URL. Values for the "applies_to" attribute are vor (version of record),am
-                (accepted manuscript), and tdm (text and data mining).</xsd:documentation>
+            <xsd:documentation>Accommodates deposit of license metadata. The license_ref value will be a URL. Values for the "applies_to" attribute are vor (version of record),am (accepted manuscript), tdm (text and data mining), and stm-asf (STM Article Sharing Framework license).</xsd:documentation>
         </xsd:annotation>
         <xsd:complexType>
             <xsd:sequence>
@@ -42,6 +39,7 @@
                                 <xsd:enumeration value="vor"/>
                                 <xsd:enumeration value="am"/>
                                 <xsd:enumeration value="tdm"/>
+                                <xsd:enumeration value="stm-asf"/>
                             </xsd:restriction>
                         </xsd:simpleType>
                     </xsd:attribute>