Ready to Learn?Ex Libris products all provide open APIs

Tech Blog

 

NFS Pair Tree Storage Plug-In

Opher Kutner on January 30th, 2014

Overview

Rosetta allows using different storage solutions, implemented as instances of the Storage Handler plug-in. Each plug-in instance uses a list of methods that can implement HTTP requests for exchanging files, between Rosetta and the storage layer.

The plug-in parameters are the unique attributes of each storage type (e.g. bucket name. secret access key in S3)

The storage entity ‘inherits’ the plug-in parameters with their default values and users can create multiple storages with different parameters (e.g. different buckets in S3)

 

NFS Storage Handler

The NFS Storage Handler comes out-of-the-box and it is in use as the default storage solution. The following example shows how a new instance of this plug-in can be created.

This example implements a different algorithm for structuring the files within folders:

The files will be located in a folder that is created based on an MD5 hash of their PID.
For example, the file that its PID is FL1234 will be located under:

root/30/2b/12/da/77/52/cc/3b/ad/78/16/8e/8a/be/34/e7

File that its PID is FL1235 will be located under:

root/f1/8e/f0/0e/ab/bd/4a/23/2e/95/ee/1c/97/fa/f3/46

This structure makes it easier on the search algorithms to bring files as they are better spread in the file systems.

 

Plugin Code

package com.exlibris.dps.repository.plugin.storage.nfs;



import java.io.File;

import java.io.FileOutputStream;

import java.io.InputStream;


import com.exlibris.core.infra.common.util.IOUtil;

import com.exlibris.core.sdk.storage.containers.StoredEntityMetaData;

import com.exlibris.digitool.common.storage.Fixity;

import com.exlibris.digitool.infrastructure.utils.Checksummer;


public class NFSPairTreeStoragePlugin extends NFSStoragePlugin {


    private static final String DIR_ROOT = "DIR_ROOT";


    public NFSPairTreeStoragePlugin() {

        super();

    }



    @Override

    public String storeEntity(InputStream is, StoredEntityMetaData storedEntityMetadata) throws Exception {


        String fileName = createFileName(storedEntityMetadata);


        String relativeDirectoryPath = getStreamRelativePath(storedEntityMetadata);

        File destFile = getStreamDirectory(relativeDirectoryPath, fileName);


        // better move/link

        if (canHandleSourcePath(storedEntityMetadata.getCurrentFilePath())) {

            is.close(); // close input stream so that 'move' can work, we don't use it anyway

            copyStream(storedEntityMetadata.getCurrentFilePath(), destFile.getAbsolutePath());

        }

        // default way - copy from input stream

        else {

            IOUtil.copy(is, new FileOutputStream(destFile));

        }

        String storedEntityIdentifier = relativeDirectoryPath + File.separator + fileName;


        if(!checkFixity(storedEntityMetadata.getFixities(), storedEntityIdentifier)) {

            deleteEntity(storedEntityIdentifier); // delete corrupt files

            return null;

        }


        // return only relative (not absolute) path

        return storedEntityIdentifier;

    }



    private String getStreamRelativePath(StoredEntityMetaData storedEntityMetaData ) throws Exception {


        String relativeDirectoryPath = File.separator;

        String pid = storedEntityMetaData.getEntityPid();

        Checksummer checksummer = new Checksummer(pid, true, false, false);

        Fixity fixity = new Fixity(Fixity.FixityAlgorithm.MD5.toString(), checksummer.getMD5());

        String value = fixity.getValue();


        for(int i=0 ; i<value.length() && i<32 ; i= i+2) {

            if(i+1 < value.length()) {

                relativeDirectoryPath += value.substring(i, i+2) + File.separator;

            } else {

                relativeDirectoryPath += value.substring(i, i+1) + File.separator;

            }

        }


        if(32 < value.length()) {

            relativeDirectoryPath += value.substring(32);

        }


        return relativeDirectoryPath;

    }


    private File getStreamDirectory(String path, String fileName) {


        File newDir = new File(parameters.get(DIR_ROOT) + File.separator + path);

        newDir.mkdirs();

        return new File(newDir.getAbsolutePath() + File.separator + fileName);

    }


    private boolean canHandleSourcePath(String srcPath) {

        try {

            File file = new File(srcPath);

            return file.canRead();

        }

        catch (Exception e) {

            return false;

        }

    }


}

XML

<pl:metadata-config xmlns:pl="http://www.exlibrisgroup.com/Plugins/1.0/" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" >

  <pl:pluginTypeName>StoragePlugin</pl:pluginTypeName>

  <pl:deployName>NFSPairTreeStoragePlugin</pl:deployName>

 <pl:className>com.exlibris.dps.repository.plugin.storage.nfs.NFSPairTreeStoragePlugin</pl:className>

  <pl:initParameters>

  <fr:x_form  xmlns:fr="http://com/exlibris/digitool/common/forms/xmlbeans">

    <form_name>NFSPairTreeStoragePlugin_Params</form_name>

    <description>NFSPairTreeStoragePlugin_parameters_form</description>

    <version>1.0</version>

    <grid_x>1</grid_x>

    <md_format>1</md_format>

        <x_fields>

        <x_field>

            <field_name>DIR_ROOT</field_name>

            <label>ui.storage.nfs.rootDir</label>

            <ui_tool_tip>ui.storage.nfs.rootDir.tooltip</ui_tool_tip>

            <single>false</single>

            <mandatory>true</mandatory>

            <x_logic_type>String</x_logic_type>

            <x_ui_type>TextField</x_ui_type>

            <css_class>width40</css_class>

            <x_options></x_options>

        </x_field>

        <x_field>

            <field_name>MAX_SIZE_IN_MB</field_name>

            <label>ui.storage.nfs.storageMaxCapacity</label>

            <ui_tool_tip>ui.storage.nfs.storageMaxCapacity.tooltip</ui_tool_tip>

            <single>false</single>

            <mandatory>true</mandatory>

            <validator_class_name>com.exlibris.core.infra.web.utils.formBuilder.validator.NumericFormValidator</validator_class_name>

            <x_logic_type>String</x_logic_type>

            <x_ui_type>TextField</x_ui_type>

            <css_class>width40</css_class>

            <x_options></x_options>

        </x_field>

        <x_field>

            <field_name>BLOCK_SIZE</field_name>

            <label>ui.storage.nfs.fileBlockSize</label>

            <ui_tool_tip>ui.storage.nfs.fileBlockSize</ui_tool_tip>

            <single>false</single>

            <default_value>8</default_value>

            <mandatory>true</mandatory>

            <validator_class_name>com.exlibris.core.infra.web.utils.formBuilder.validator.NumericFormValidator</validator_class_name>

            <x_logic_type>String</x_logic_type>

            <x_ui_type>TextField</x_ui_type>

            <css_class>width40</css_class>

            <x_options></x_options>

        </x_field>

        <x_field>

            <field_name>FILES_HANDLING_METHOD</field_name>

            <label>ui.storage.nfs.filesHandlingMethod</label>

            <ui_tool_tip>filesHandlingMethod.filesHandlingMethod</ui_tool_tip>

            <single>false</single>

            <default_value>copy</default_value>

            <pos_x>1</pos_x>

            <pos_y>3</pos_y>

            <mandatory>true</mandatory>

            <x_logic_type>String</x_logic_type>

            <x_ui_type>Radio</x_ui_type>

            <populator_class_name>com.exlibris.core.infra.web.utils.formBuilder.populators.CodeTablePopulator</populator_class_name>

            <x_populator_parameters>

                <x_populator_parameter>

                    <parameter>table</parameter>

                    <value>FILES_HANDLING_METHOD_CODE_TABLE_PLACE_HOLDER</value>

                </x_populator_parameter>

            </x_populator_parameters>

            <x_options></x_options>

        </x_field>

    </x_fields>

</fr:x_form>

</pl:initParameters>

  <pl:description>NFS Pair Tree Storage Plugin</pl:description>

  <pl:version>1.1</pl:version>

  <pl:materialType>DIGITAL</pl:materialType>

  <pl:module>Repository</pl:module>

  <pl:generalType>TASK</pl:generalType>

  <pl:publicApi>N</pl:publicApi>

  <pl:implType>java</pl:implType>

  <pl:contactPerson>

    <contact_info xmlns="http://www.exlibrisgroup.com/Plugins/1.0/">

      <contactType>admin</contactType>

      <firstName>Exlibris</firstName>

      <lastName>Ltd</lastName>

      <telephone1>(050)746-1234</telephone1>

      <telephone2></telephone2>

      <email>shaib@exlibris.co.il</email>

      <address1>Agodat Asport2 Building 9</address1>

      <address2></address2>

      <city>Jerusalem</city>

      <zipCode>00000</zipCode>

      <country>Israel</country>

    </contact_info>

  </pl:contactPerson>

</pl:metadata-config>