text:
/*
 * Early versions of this class were inspired by Jhead, a C program for extracting and
 * manipulating the Exif data within files written by Matthias Wandel.
 *   http://www.sentex.net/~mwandel/jhead/
 *
 * Jhead is public domain software - that is, you can do whatever you want
 * with it, and include it software that is licensed under the GNU or the
 * BSD license, or whatever other licence you choose, including proprietary
 * closed source licenses.  Similarly, I release this Java version under the
 * same license, though I do ask that you leave this header in tact.
 *
 * If you make modifications to this code that you think would benefit the
 * wider community, please send me a copy and I'll post it on my site.  Unlike
 * Jhead, this code (as it stands) only supports reading of Exif data - no
 * manipulation.
 *
 * If you make use of this code, I'd appreciate hearing about it.
 *   drew.noakes@drewnoakes.com
 * Latest version of this software kept at
 *   http://drewnoakes.com/
 *
 * Created on 28 April 2002, 23:54
 * Modified 04 Aug 2002
 * - Renamed constants to be inline with changes to ExifTagValues interface
 * - Substituted usage of JDK 1.4 features (java.nio package)
 * Modified 29 Oct 2002 (v1.2)
 * - Proper traversing of Exif file structure and complete refactor & tidy of
 *   the codebase (a few unnoticed bugs removed)
 * - Reads makernote data for 6 families of camera (5 makes)
 * - Tags now stored in directories... use the IFD_* constants to refer to the
 *   image file directory you require (Exif, Interop, GPS and Makernote*) --
 *   this avoids collisions where two tags share the same code
 * - Takes componentCount of unknown tags into account
 * - Now understands GPS tags (thanks to Colin Briton for his help with this)
 * - Some other bug fixes, pointed out by users around the world.  Thanks!
 * Modified 27 Nov 2002 (v2.0)
 * - Renamed to ExifReader
 * - Moved to new package sage.media.exif.metadata.exif
 * Modified since, however changes have not been logged.  See release notes for
 * library-wide modifications.
 */
package sage.media.exif.metadata.exif;

import java.io.File;
import java.io.InputStream;
import java.util.HashMap;
import java.util.Iterator;
import java.util.LinkedList;
import java.util.List;

import sage.media.exif.imaging.jpeg.JpegProcessingException;
import sage.media.exif.imaging.jpeg.JpegSegmentData;
import sage.media.exif.imaging.jpeg.JpegSegmentReader;
import sage.media.exif.lang.Rational;
import sage.media.exif.metadata.Directory;
import sage.media.exif.metadata.Metadata;
import sage.media.exif.metadata.MetadataException;
import sage.media.exif.metadata.MetadataReader;

/**
 * Decodes Exif binary data, populating a Metadata object with
 * tag values in ExifDirectory, GpsDirectory and
 * one of the many camera makernote directories.
 *
 * @author Drew Noakes http://drewnoakes.com
 */
public class ExifReader implements MetadataReader {
    /**
     * The number of bytes used per format descriptor.
     */
    private static final int[] BYTES_PER_FORMAT = { 0, 1, 1, 2, 4, 8, 1, 1, 2,
            4, 8, 4, 8 };

    /**
     * The number of formats known.
     */
    private static final int MAX_FORMAT_CODE = 12;

    // Format types
    // Note: Cannot use the DataFormat enumeration in the case statement that
    // uses these tags.
    // Is there a better way?
    private static final int FMT_BYTE = 1;
    private static final int FMT_STRING = 2;
    private static final int FMT_USHORT = 3;
    private static final int FMT_ULONG = 4;
    private static final int FMT_URATIONAL = 5;
    private static final int FMT_SBYTE = 6;
    private static final int FMT_UNDEFINED = 7;
    private static final int FMT_SSHORT = 8;
    private static final int FMT_SLONG = 9;
    private static final int FMT_SRATIONAL = 10;
    private static final int FMT_SINGLE = 11;
    private static final int FMT_DOUBLE = 12;

    public static final int TAG_EXIF_OFFSET = 0x8769;
    public static final int TAG_INTEROP_OFFSET = 0xA005;
    public static final int TAG_GPS_INFO_OFFSET = 0x8825;
    public static final int TAG_MAKER_NOTE = 0x927C;

    public static final int TIFF_HEADER_START_OFFSET = 6;

    /**
     * The Exif segment as an array of bytes.
     */
    private final byte[] _data;

    /**
     * Represents the native byte ordering used in the JPEG segment. If true,
     * then we're using Motorolla ordering (Big endian), else we're using Intel
     * ordering (Little endian).
     */
    private boolean _isMotorollaByteOrder;

    /**
     * Bean instance to store information about the image and
     * camera/scanner/capture device.
     */
    private Metadata _metadata;

    /**
     * The Exif directory used (loaded lazily)
     */
    private ExifDirectory _exifDirectory = null;

    /**
     * Creates an ExifReader for a JpegSegmentData object.
     *
     * @param segmentData
     * @deprecated Not all files will be Jpegs! This overload doesn't offer much
     *             convenience to the caller.
     */
    public ExifReader(JpegSegmentData segmentData) {
        // TODO consider removing this constructor and requiring callers to pass
        // a byte[] or other means to read the exif segment in isolation... not
        // all files will be Jpegs!
        this(segmentData.getSegment(JpegSegmentReader.SEGMENT_APP1));
    }

    /**
     * Creates an ExifReader for a Jpeg jpegFile.
     *
     * @param jpegFile
     * @throws JpegProcessingException
     * @deprecated Not all files will be Jpegs! Use a constructor that provides
     *             the exif segment in isolation.
     */
    public ExifReader(File jpegFile) throws JpegProcessingException {
        // TODO consider removing this constructor and requiring callers to pass
        // a byte[] or other means to read the exif segment in isolation... not
        // all files will be Jpegs!
        this(new JpegSegmentReader(jpegFile)
                .readSegment(JpegSegmentReader.SEGMENT_APP1));
    }

    /**
     * Creates an ExifReader for a Jpeg stream.
     *
     * @param jpegInputStream
     *            JPEG stream. Stream will be closed.
     * @deprecated Not all files will be Jpegs! Use a constructor that provides
     *             the exif segment in isolation.
     */
    public ExifReader(InputStream jpegInputStream)
            throws JpegProcessingException {
        // TODO consider removing this constructor and requiring callers to pass
        // a byte[] or other means to read the exif segment in isolation... not
        // all files will be Jpegs!
        this(new JpegSegmentReader(jpegInputStream)
                .readSegment(JpegSegmentReader.SEGMENT_APP1));
    }

    /**
     * Creates an ExifReader for the given Exif data segment.
     */
    public ExifReader(byte[] data) {
        _data = data;
    }

    /**
     * Performs the Exif data extraction, returning a new instance of
     * Metadata.
     */
    public Metadata extract() {
        return extract(new Metadata());
    }

    /**
     * Performs the Exif data extraction, adding found values to the specified
     * instance of Metadata.
     */
    public Metadata extract(Metadata metadata) {
        _metadata = metadata;
        if (_data == null)
            return _metadata;

        ExifDirectory directory = getExifDirectory();
        // check for the header length
        if (_data.length <= 14) {
            directory
                    .addError("Exif data segment must contain at least 14 bytes");
            return _metadata;
        }

        if (!isExifSegment(_data)) {
            // check for the header preamble
            directory.addError("Exif data segment doesn't begin with 'Exif'");
            return _metadata;
        }

        return extractIFD(_metadata, TIFF_HEADER_START_OFFSET);

    }

    public static boolean isExifSegment(byte[] data) {

        if (data == null || data.length <= 14) {
            return false;
        }
        if (!"Exif\0\0".equals(new String(data, 0, 6))) {
            return false;
        }
        return true;
    }

    private ExifDirectory getExifDirectory() {
        if (_exifDirectory == null) {
            _exifDirectory = (ExifDirectory) _metadata
                    .getDirectory(ExifDirectory.class);
        }
        return _exifDirectory;
    }

    /**
     * Performs the Exif data extraction on a Tiff/Raw, adding found values to
     * the specified instance of Metadata.
     */
    public Metadata extractTiff(Metadata metadata) {
        return extractIFD(metadata, 0);
    }

    private Metadata extractIFD(Metadata metadata, int tiffHeaderOffset) {
        _metadata = metadata;
        if (_data == null)
            return _metadata;

        ExifDirectory directory = getExifDirectory();

        // this should be either "MM" or "II"
        String byteOrderIdentifier = new String(_data, tiffHeaderOffset, 2);
        if (!setByteOrder(byteOrderIdentifier)) {
            directory
                    .addError("Unclear distinction between Motorola/Intel byte ordering: "
                            + byteOrderIdentifier);
            return _metadata;
        }

        // Check the next two values for correctness.
        if (get16Bits(2 + tiffHeaderOffset) != 0x2a) {
            directory
                    .addError("Invalid Exif start - should have 0x2A at offset 8 in Exif header");
            return _metadata;
        }

        int firstDirectoryOffset = get32Bits(4 + tiffHeaderOffset)
                + tiffHeaderOffset;

        // David Ekholm sent an digital camera image that has this problem
        if (firstDirectoryOffset >= _data.length - 1) {
            directory
                    .addError("First exif directory offset is beyond end of Exif data segment");
            // First directory normally starts 14 bytes in -- try it here and
            // catch another error in the worst case
            firstDirectoryOffset = 14;
        }

        HashMap processedDirectoryOffsets = new HashMap();

        // 0th IFD (we merge with Exif IFD)
        processDirectory(directory, processedDirectoryOffsets,
                firstDirectoryOffset, tiffHeaderOffset,0);

        // after the extraction process, if we have the correct tags, we may be
        // able to store thumbnail information
        storeThumbnailBytes(directory, tiffHeaderOffset);

        return _metadata;
    }

    private void storeThumbnailBytes(ExifDirectory exifDirectory,
            int tiffHeaderOffset) {
        if (!exifDirectory.containsTag(ExifDirectory.TAG_COMPRESSION))
            return;

        if (!exifDirectory.containsTag(ExifDirectory.TAG_THUMBNAIL_LENGTH)
                || !exifDirectory
                        .containsTag(ExifDirectory.TAG_THUMBNAIL_OFFSET))
            return;

        try {
            int offset = exifDirectory
                    .getInt(ExifDirectory.TAG_THUMBNAIL_OFFSET);
            int length = exifDirectory
                    .getInt(ExifDirectory.TAG_THUMBNAIL_LENGTH);
            byte[] result = new byte[length];
            for (int i = 0; i < result.length; i++) {
                result[i] = _data[tiffHeaderOffset + offset + i];
            }
            exifDirectory
                    .setByteArray(ExifDirectory.TAG_THUMBNAIL_DATA, result);
        } catch (Throwable e) {
            exifDirectory.addError("Unable to extract thumbnail: "
                    + e.getMessage());
        }
    }

    /**
     * Sets the _isMotorollaByteOrder flag to true or false, depending upon the
     * file's byte order string. If the string cannot be interpreted, false is
     * returned.
     *
     * @param byteOrderIdentifier
     *            a two-character string; either "MM" for Motorolla or "II" for
     *            Intel.
     * @return true if successful, otherwise false.
     */
    private boolean setByteOrder(String byteOrderIdentifier) {
        if ("MM".equals(byteOrderIdentifier)) {
            _isMotorollaByteOrder = true;
        } else if ("II".equals(byteOrderIdentifier)) {
            _isMotorollaByteOrder = false;
        } else {
            return false;
        }
        return true;
    }

    /**
     * Process one of the nested Tiff IFD directories. 2 bytes: number of tags
     * for each tag 2 bytes: tag type 2 bytes: format code 4 bytes: component
     * count
     */
    private void processDirectory(Directory directory,
            HashMap processedDirectoryOffsets, int dirStartOffset,
            int tiffHeaderOffset, int offsetSchema) {

    	List makerNoteOffsets=new LinkedList();

    	boolean switchedByteOrder=false;

    	do {

	        // check for directories we've already visited to avoid stack overflows
	        // when recursive/cyclic directory structures exist
	        if (processedDirectoryOffsets.containsKey(new Integer(dirStartOffset)))
	            break;

	        // remember that we've visited this directory so that we don't visit it
	        // again later
	        processedDirectoryOffsets.put(new Integer(dirStartOffset), "processed");

	        if (dirStartOffset >= _data.length || dirStartOffset < 0) {
	            directory
	                    .addError("Ignored directory marked to start outside data segement");
	            break;
	        }

	        if (! isDirectoryLengthValid(dirStartOffset, tiffHeaderOffset)) {
	            /**
	             * some tools (windows vista) modify the byte order for the EXIF data,
	             * but copy the MakerNote bytes without switching byte order.
	             * This flag temporarily switches byte order while processing a makernote dir
	             */
	        	_isMotorollaByteOrder=!_isMotorollaByteOrder;
	        	switchedByteOrder=true;
	        	if (! isDirectoryLengthValid(dirStartOffset, tiffHeaderOffset)) {
		            directory.addError("Illegally sized directory");
		            break;
	        	}
	        }

	        // First two bytes in the IFD are the number of tags in this directory
	        int dirTagCount = get16Bits(dirStartOffset);

	        // Handle each tag in this directory
	        for (int tagNumber = 0; tagNumber < dirTagCount; tagNumber++) {
	            final int tagOffset = calculateTagOffset(dirStartOffset, tagNumber);

	            // 2 bytes for the tag type
	            final int tagType = get16Bits(tagOffset);

	            // 2 bytes for the format code
	            final int formatCode = get16Bits(tagOffset + 2);
	            if (formatCode < 1 || formatCode > MAX_FORMAT_CODE) {
	                directory.addError("Invalid format code: " + formatCode);
	                continue;
	            }

	            // 4 bytes dictate the number of components in this tag's data
	            final int componentCount = get32Bits(tagOffset + 4);
	            if (componentCount < 0) {
	                directory.addError("Negative component count in EXIF");
	                continue;
	            }
	            // each component may have more than one byte... calculate the total
	            // number of bytes
	            final int byteCount = componentCount * BYTES_PER_FORMAT[formatCode];
	            final int tagValueOffset = calculateTagValueOffset(byteCount,
	                    tagOffset, tiffHeaderOffset,offsetSchema);
	            if (tagValueOffset < 0 || tagValueOffset > _data.length) {
	                directory.addError("Illegal pointer offset value in EXIF");
	                continue;
	            }

	            // Check that this tag isn't going to allocate outside the bounds of
	            // the data array.
	            // This addresses an uncommon OutOfMemoryError.
	            if (byteCount < 0 || tagValueOffset + byteCount > _data.length) {
	                directory.addError("Illegal number of bytes: " + byteCount);
	                continue;
	            }

	            // Calculate the value as an offset for cases where the tag
	            // represents directory
	            final int subdirOffset = tiffHeaderOffset
	                    + get32Bits(tagValueOffset);

	            switch (tagType) {
	            case TAG_EXIF_OFFSET:
	                processDirectory(_metadata.getDirectory(ExifDirectory.class),
	                        processedDirectoryOffsets, subdirOffset,
	                        tiffHeaderOffset,offsetSchema);
	                continue;
	            case TAG_INTEROP_OFFSET:
	                processDirectory(_metadata
	                        .getDirectory(ExifInteropDirectory.class),
	                        processedDirectoryOffsets, subdirOffset,
	                        tiffHeaderOffset,offsetSchema);
	                continue;
	            case TAG_GPS_INFO_OFFSET:
	                processDirectory(_metadata.getDirectory(GpsDirectory.class),
	                        processedDirectoryOffsets, subdirOffset,
	                        tiffHeaderOffset,offsetSchema);
	                continue;
	            case TAG_MAKER_NOTE:
	            	makerNoteOffsets.add(new Integer(tagValueOffset));
	                continue;
	            default:
	                processTag(directory, tagType, tagValueOffset, componentCount,
	                        formatCode);
	                break;
	            }
	        }

	        // at the end of each IFD is an optional link to the next IFD
	        final int finalTagOffset = calculateTagOffset(dirStartOffset,
	                dirTagCount);
	        int nextDirectoryOffset = get32Bits(finalTagOffset);
	        if (nextDirectoryOffset != 0) {
	            nextDirectoryOffset += tiffHeaderOffset;
	            if (nextDirectoryOffset >= _data.length) {
	                // Last 4 bytes of IFD reference another IFD with an address
	                // that is out of bounds
	                // Note this could have been caused by jhead 1.3 cropping too
	                // much
	                break;
	            } else if (nextDirectoryOffset < dirStartOffset) {
	                // Last 4 bytes of IFD reference another IFD with an address
	                // that is before the start of this directory
	                break;
	            }
	            // the next directory is of same type as this one
	        }
	        dirStartOffset=nextDirectoryOffset;
    	} while ( dirStartOffset!=0 );

    	// process makerNotes
    	if ( offsetSchema==0){
	    	try {
	    		offsetSchema=directory.getInt(ExifDirectory.TAG_OFFSET_SCHEMA);
	    	} catch (MetadataException e){
	    		offsetSchema=0;
	    	}
    	}

    	for (Iterator it = makerNoteOffsets.iterator(); it
				.hasNext();) {
			Integer offset = (Integer) it.next();
	        processMakerNote(offset.intValue(), processedDirectoryOffsets,
	                tiffHeaderOffset,offsetSchema);
		}

    	if ( switchedByteOrder)
    		_isMotorollaByteOrder=!_isMotorollaByteOrder;
    }

    private void processMakerNote(int subdirOffset,
            HashMap processedDirectoryOffsets, int tiffHeaderOffset, int offsetSchema) {
        // Determine the camera model and makernote format
        Directory exifDirectory = _metadata.getDirectory(ExifDirectory.class);

        if (exifDirectory == null)
            return;

        String cameraModel = exifDirectory.getString(ExifDirectory.TAG_MAKE);
        final String firstTwoChars = new String(_data, subdirOffset, 2);
        final String firstThreeChars = new String(_data, subdirOffset, 3);
        final String firstFourChars = new String(_data, subdirOffset, 4);
        final String firstFiveChars = new String(_data, subdirOffset, 5);
        final String firstSixChars = new String(_data, subdirOffset, 6);
        final String firstSevenChars = new String(_data, subdirOffset, 7);
        final String firstEightChars = new String(_data, subdirOffset, 8);
        if ("OLYMP".equals(firstFiveChars) || "EPSON".equals(firstFiveChars)
                || "AGFA".equals(firstFourChars)) {
            // Olympus Makernote
            // Epson and Agfa use Olypus maker note standard, see:
            // http://www.ozhiker.com/electronics/pjmt/jpeg_info/
            processDirectory(_metadata
                    .getDirectory(OlympusMakernoteDirectory.class),
                    processedDirectoryOffsets, subdirOffset + 8,
                    tiffHeaderOffset,offsetSchema);
        } else if (cameraModel != null
                && cameraModel.trim().toUpperCase().startsWith("NIKON")) {
            if ("Nikon".equals(firstFiveChars)) {
            	/*
            	 * There are two scenarios here:
            	 *
            	 * Type 1: **
            	 *
            	 * :0000: 4E 69 6B 6F 6E 00 01 00-05 00 02 00 02 00 06 00 Nikon...........
            	 * :0010: 00 00 EC 02 00 00 03 00-03 00 01 00 00 00 06 00 ................
            	 *
            	 * Type 3: **
            	 *
            	 * :0000: 4E 69 6B 6F 6E 00 02 00-00 00 4D 4D 00 2A 00 00 Nikon....MM.*...
            	 * :0010: 00 08 00 1E 00 01 00 07-00 00 00 04 30 32 30 30 ............0200
            	 */
                if (_data[subdirOffset + 6] == 1)
                    processDirectory(_metadata
                            .getDirectory(NikonType1MakernoteDirectory.class),
                            processedDirectoryOffsets, subdirOffset + 8,
                            tiffHeaderOffset,offsetSchema);
                else if (_data[subdirOffset + 6] == 2)
                    processDirectory(_metadata
                            .getDirectory(NikonType2MakernoteDirectory.class),
                            processedDirectoryOffsets, subdirOffset + 18,
                            subdirOffset + 10,offsetSchema);
                else
                    exifDirectory
                            .addError("Unsupported makernote data ignored.");
            } else {
                // The IFD begins with the first MakerNote byte (no ASCII name).
                // This occurs with CoolPix 775, E990 and D1 models.
                processDirectory(_metadata
                        .getDirectory(NikonType2MakernoteDirectory.class),
                        processedDirectoryOffsets, subdirOffset,
                        tiffHeaderOffset,offsetSchema);
            }
        } else if ("SONY CAM".equals(firstEightChars)
                || "SONY DSC".equals(firstEightChars)) {
            processDirectory(_metadata
                    .getDirectory(SonyMakernoteDirectory.class),
                    processedDirectoryOffsets, subdirOffset + 12,
                    tiffHeaderOffset,offsetSchema);
        } else if ("KDK".equals(firstThreeChars)) {
            processDirectory(_metadata
                    .getDirectory(KodakMakernoteDirectory.class),
                    processedDirectoryOffsets, subdirOffset + 20,
                    tiffHeaderOffset,offsetSchema);
        } else if ("Canon".equalsIgnoreCase(cameraModel)) {
            processDirectory(_metadata
                    .getDirectory(CanonMakernoteDirectory.class),
                    processedDirectoryOffsets, subdirOffset, tiffHeaderOffset,offsetSchema);
        } else if (cameraModel != null
                && cameraModel.toUpperCase().startsWith("CASIO")) {
            if ("QVC\u0000\u0000\u0000".equals(firstSixChars))
                processDirectory(_metadata
                        .getDirectory(CasioType2MakernoteDirectory.class),
                        processedDirectoryOffsets, subdirOffset + 6,
                        tiffHeaderOffset,offsetSchema);
            else
                processDirectory(_metadata
                        .getDirectory(CasioType1MakernoteDirectory.class),
                        processedDirectoryOffsets, subdirOffset,
                        tiffHeaderOffset,offsetSchema);
        } else if ("FUJIFILM".equals(firstEightChars)
                || "Fujifilm".equalsIgnoreCase(cameraModel)) {
            // TODO make this field a passed parameter, to avoid threading
            // issues
            boolean byteOrderBefore = _isMotorollaByteOrder;
            // bug in fujifilm makernote ifd means we temporarily use Intel byte
            // ordering
            _isMotorollaByteOrder = false;
            // the 4 bytes after "FUJIFILM" in the makernote point to the start
            // of the makernote
            // IFD, though the offset is relative to the start of the makernote,
            // not the TIFF
            // header (like everywhere else)
            int ifdStart = subdirOffset + get32Bits(subdirOffset + 8);
            processDirectory(_metadata
                    .getDirectory(FujifilmMakernoteDirectory.class),
                    processedDirectoryOffsets, ifdStart, tiffHeaderOffset,offsetSchema);
            _isMotorollaByteOrder = byteOrderBefore;
        } else if (cameraModel != null
                && cameraModel.toUpperCase().startsWith("MINOLTA")) {
            // Cases seen with the model starting with MINOLTA in capitals seem
            // to have a valid Olympus makernote
            // area that commences immediately.
            processDirectory(_metadata
                    .getDirectory(OlympusMakernoteDirectory.class),
                    processedDirectoryOffsets, subdirOffset, tiffHeaderOffset,offsetSchema);
        } else if ("KC".equals(firstTwoChars) || "MINOL".equals(firstFiveChars)
                || "MLY".equals(firstThreeChars)
                || "+M+M+M+M".equals(firstEightChars)) {
            // This Konica data is not understood. Header identified in
            // accordance with information at this site:
            // http://www.ozhiker.com/electronics/pjmt/jpeg_info/minolta_mn.html
            // TODO determine how to process the information described at the
            // above website
            exifDirectory.addError("Unsupported Konica/Minolta data ignored.");
        } else if ("KYOCERA".equals(firstSevenChars)) {
            // http://www.ozhiker.com/electronics/pjmt/jpeg_info/kyocera_mn.html
            processDirectory(_metadata
                    .getDirectory(KyoceraMakernoteDirectory.class),
                    processedDirectoryOffsets, subdirOffset + 22,
                    tiffHeaderOffset,offsetSchema);
        } else if ("Panasonic\u0000\u0000\u0000".equals(new String(_data,
                subdirOffset, 12))) {
            // NON-Standard TIFF IFD Data using Panasonic Tags. There is no
            // Next-IFD pointer after the IFD
            // Offsets are relative to the start of the TIFF header at the
            // beginning of the EXIF segment
            // more information here:
            // http://www.ozhiker.com/electronics/pjmt/jpeg_info/panasonic_mn.html
            processDirectory(_metadata
                    .getDirectory(PanasonicMakernoteDirectory.class),
                    processedDirectoryOffsets, subdirOffset + 12,
                    tiffHeaderOffset,offsetSchema);
        } else if ("AOC\u0000".equals(firstFourChars)) {
            // NON-Standard TIFF IFD Data using Casio Type 2 Tags
            // IFD has no Next-IFD pointer at end of IFD, and
            // Offsets are relative to the start of the current IFD tag, not the
            // TIFF header
            // Observed for:
            // - Pentax ist D
            processDirectory(_metadata
                    .getDirectory(CasioType2MakernoteDirectory.class),
                    processedDirectoryOffsets, subdirOffset + 6, subdirOffset,offsetSchema);
        } else if (cameraModel != null
                && (cameraModel.toUpperCase().startsWith("PENTAX") || cameraModel
                        .toUpperCase().startsWith("ASAHI"))) {
            // NON-Standard TIFF IFD Data using Pentax Tags
            // IFD has no Next-IFD pointer at end of IFD, and
            // Offsets are relative to the start of the current IFD tag, not the
            // TIFF header
            // Observed for:
            // - PENTAX Optio 330
            // - PENTAX Optio 430
            processDirectory(_metadata
                    .getDirectory(PentaxMakernoteDirectory.class),
                    processedDirectoryOffsets, subdirOffset, subdirOffset,offsetSchema);
        } else {
            // TODO how to store makernote data when it's not from a supported
            // camera model?
            // this is difficult as the starting offset is not known. we could
            // look for it...
            exifDirectory.addError("Unsupported makernote data ignored.");
        }
    }

    private boolean isDirectoryLengthValid(int dirStartOffset,
            int tiffHeaderOffset) {
        int dirTagCount = get16Bits(dirStartOffset);
        int dirLength = (2 + (12 * dirTagCount) + 4);
        if (dirLength + dirStartOffset + tiffHeaderOffset >= _data.length) {
            // Note: Files that had thumbnails trimmed with jhead 1.3 or earlier
            // might trigger this
            return false;
        }
        return true;
    }

    private void processTag(Directory directory, int tagType,
            int tagValueOffset, int componentCount, int formatCode) {
        // Directory simply stores raw values
        // The display side uses a Descriptor class per directory to turn the
        // raw values into 'pretty' descriptions
        switch (formatCode) {
        case FMT_UNDEFINED:
            // this includes exif user comments
            final byte[] tagBytes = new byte[componentCount];
            final int byteCount = componentCount * BYTES_PER_FORMAT[formatCode];
            for (int i = 0; i < byteCount; i++)
                tagBytes[i] = _data[tagValueOffset + i];
            directory.setByteArray(tagType, tagBytes);
            break;
        case FMT_STRING:
            directory.setString(tagType, readString(tagValueOffset,
                    componentCount));
            break;
        case FMT_SRATIONAL:
        case FMT_URATIONAL:
            if (componentCount == 1) {
                Rational rational = new Rational(get32Bits(tagValueOffset),
                        get32Bits(tagValueOffset + 4));
                directory.setRational(tagType, rational);
            } else {
                Rational[] rationals = new Rational[componentCount];
                for (int i = 0; i < componentCount; i++)
                    rationals[i] = new Rational(get32Bits(tagValueOffset
                            + (8 * i)), get32Bits(tagValueOffset + 4 + (8 * i)));
                directory.setRationalArray(tagType, rationals);
            }
            break;
        case FMT_SBYTE:
        case FMT_BYTE:
            if (componentCount == 1) {
                // this may need to be a byte, but I think casting to int is
                // fine
                int b = _data[tagValueOffset];
                directory.setInt(tagType, b);
            } else {
                int[] bytes = new int[componentCount];
                for (int i = 0; i < componentCount; i++)
                    bytes[i] = _data[tagValueOffset + i];
                directory.setIntArray(tagType, bytes);
            }
            break;
        case FMT_SINGLE:
        case FMT_DOUBLE:
            if (componentCount == 1) {
                int i = _data[tagValueOffset];
                directory.setInt(tagType, i);
            } else {
                int[] ints = new int[componentCount];
                for (int i = 0; i < componentCount; i++)
                    ints[i] = _data[tagValueOffset + i];
                directory.setIntArray(tagType, ints);
            }
            break;
        case FMT_USHORT:
        case FMT_SSHORT:
            if (componentCount == 1) {
                int i = get16Bits(tagValueOffset);
                directory.setInt(tagType, i);
            } else {
                int[] ints = new int[componentCount];
                for (int i = 0; i < componentCount; i++)
                    ints[i] = get16Bits(tagValueOffset + (i * 2));
                directory.setIntArray(tagType, ints);
            }
            break;
        case FMT_SLONG:
        case FMT_ULONG:
            if (componentCount == 1) {
                int i = get32Bits(tagValueOffset);
                directory.setInt(tagType, i);
            } else {
                int[] ints = new int[componentCount];
                for (int i = 0; i < componentCount; i++)
                    ints[i] = get32Bits(tagValueOffset + (i * 4));
                directory.setIntArray(tagType, ints);
            }
            break;
        default:
            directory.addError("Unknown format code " + formatCode
                    + " for tag " + tagType);
        }
    }

    private int calculateTagValueOffset(int byteCount, int dirEntryOffset,
            int tiffHeaderOffset, int offsetSchema) {
        if (byteCount > 4) {
            // If its bigger than 4 bytes, the dir entry contains an offset.
            // dirEntryOffset must be passed, as some makernote implementations
            // (e.g. FujiFilm) incorrectly use an
            // offset relative to the start of the makernote itself, not the
            // TIFF segment.
            final int offsetVal = get32Bits(dirEntryOffset + 8)+offsetSchema;
            if (offsetVal + byteCount > _data.length) {
                // Bogus pointer offset and / or bytecount value
                return -1; // signal error
            }
            return tiffHeaderOffset + offsetVal;
        } else {
            // 4 bytes or less and value is in the dir entry itself
            return dirEntryOffset + 8;
        }
    }

    /**
     * Creates a String from the _data buffer starting at the specified offset,
     * and ending where byte=='\0' or where length==maxLength.
     */
    private String readString(int offset, int maxLength) {
        int length = 0;
        while ((offset + length) < _data.length
                && _data[offset + length] != '\0' && length < maxLength)
            length++;

        return new String(_data, offset, length);
    }

    /**
     * Determine the offset at which a given InteropArray entry begins within
     * the specified IFD.
     *
     * @param dirStartOffset
     *            the offset at which the IFD starts
     * @param entryNumber
     *            the zero-based entry number
     */
    private int calculateTagOffset(int dirStartOffset, int entryNumber) {
        // add 2 bytes for the tag count
        // each entry is 12 bytes, so we skip 12 * the number seen so far
        return dirStartOffset + 2 + (12 * entryNumber);
    }

    /**
     * Get a 16 bit value from file's native byte order. Between 0x0000 and
     * 0xFFFF.
     */
    private int get16Bits(int offset) {
        if (offset < 0 || offset + 2 > _data.length)
            throw new ArrayIndexOutOfBoundsException(
                    "attempt to read data outside of exif segment (index "
                            + offset + " where max index is "
                            + (_data.length - 1) + ")");

        if (_isMotorollaByteOrder) {
            // Motorola - MSB first
            return (_data[offset] << 8 & 0xFF00) | (_data[offset + 1] & 0xFF);
        } else {
            // Intel ordering - LSB first
            return (_data[offset + 1] << 8 & 0xFF00) | (_data[offset] & 0xFF);
        }
    }

    /**
     * Get a 32 bit value from file's native byte order.
     */
    private int get32Bits(int offset) {
        if (offset < 0 || offset + 4 > _data.length)
            throw new ArrayIndexOutOfBoundsException(
                    "attempt to read data outside of exif segment (index "
                            + offset + " where max index is "
                            + (_data.length - 1) + ")");

        if (_isMotorollaByteOrder) {
            // Motorola - MSB first
            return (_data[offset] << 24 & 0xFF000000)
                    | (_data[offset + 1] << 16 & 0xFF0000)
                    | (_data[offset + 2] << 8 & 0xFF00)
                    | (_data[offset + 3] & 0xFF);
        } else {
            // Intel ordering - LSB first
            return (_data[offset + 3] << 24 & 0xFF000000)
                    | (_data[offset + 2] << 16 & 0xFF0000)
                    | (_data[offset + 1] << 8 & 0xFF00)
                    | (_data[offset] & 0xFF);
        }
    }
}


text:
import os, os.path, shutil, re, datetime

from modu.persist import Store, dbapi, sql
from modu.web import app

import modusite
from modusite.model import project, release
from modu.sites import modusite_site

RE_TARBALL_VERSION = re.compile(r'^(\w+?)-((?:\d\.)+\d.*)\.tar\.gz$', re.IGNORECASE)

def get_release_request(hostname):
	env = {
		'SERVER_NAME'		: hostname,
		'SERVER_PORT'		: '80',
		'PATH_INFO'			: '/',
		'SCRIPT_NAME' 		: '',
		'wsgi.url_scheme'	: 'https',
	}

	req = app.Request()
	application = app.get_application(env)
	if not(application):
		raise RuntimeError("Can't load application config.")

	application.make_immutable()
	req = app.configure_request(env, application)
	if(hasattr(application.site, 'configure_request')):
		application.site.configure_request(req)

	return req

def create(hostname, source, destination, nightly=False):
	os.chdir(source)
	os.system('svn update')

	existing = os.listdir(destination)

	os.system('python setup.py egg_info %s sdist' % (('-RDb ""', '')[int(nightly)]))

	filename = None
	for item in os.listdir(os.path.join(source, 'dist')):
		if(item not in existing):
			filename = item
			shutil.copy(os.path.join(source, 'dist', item), destination)
			break
	else:
		raise IOError("Tarball not found.")

	match = RE_TARBALL_VERSION.match(filename)
	if not(match):
		raise RuntimeError("Filename %s can't be parsed for project and version information." % filename)

	project_name = match.group(1)
	release_version = match.group(2)

	pool = dbapi.connect(modusite_site.db_url)
	store = Store(pool)

	store.ensure_factory('project', model_class=project.Project)
	store.ensure_factory('release', model_class=release.Release)

	p = store.load_one('project', shortname=project_name)
	if not(p):
		raise RuntimeError("No such project, '%s'" % project_name)

	r = store.load_one('release', project_id=p.get_id(), version_string=release_version)
	if(r):
		raise RuntimeError("There is already a '%s' release with version string '%s'" % (project_name, release_version))

	r = release.Release()
	r.project_id = p.get_id()
	r.active = 1
	r.nightly = int(nightly)
	r.release_date = datetime.datetime.now()

	r.license_name = p.license_name
	r.license_url = p.license_url
	r.installation_url = p.installation_url
	r.changelog_url = p.changelog_url

	r.version_string = release_version
	r.version_weight = re.sub(r'[-_.]', '', release_version)

	req = get_release_request(hostname)
	r.load_tarball_info(req, filename)

	store.save(r)

redpajama_set_name: RedPajamaGithub


text:
<Record>
<Term>Staphylococcal Vaccines</Term>
<SemanticType>Pharmacologic Substance</SemanticType>
<SemanticType>Immunologic Factor</SemanticType>
<ParentTerm>Bacterial Vaccines</ParentTerm>
<ClassificationPath>Complex Mixtures/Biological Products/Vaccines/Bacterial Vaccines/Staphylococcal Vaccines</ClassificationPath>
<BroaderTerm>Biological Products</BroaderTerm>
<BroaderTerm>Bacterial Vaccines</BroaderTerm>
<BroaderTerm>Staphylococcal Vaccines</BroaderTerm>
<BroaderTerm>Vaccines</BroaderTerm>
<BroaderTerm>Complex Mixtures</BroaderTerm>
<Synonym>Staphylococcal Vaccines</Synonym>
<Source>MeSH</Source>
</Record>


text:
<?php namespace Teepluss\Categorize;

use Closure;
use Illuminate\Config\Repository;
use Illuminate\Database\Eloquent\Collection;
use Teepluss\Categorize\Categories\ProviderInterface as CategoryProviderInterface;
use Teepluss\Categorize\CategoryRelates\ProviderInterface as CategoryRelateProviderInterface;
use Teepluss\Categorize\CategoryHierarchy\ProviderInterface as CategoryHierarchyProviderInterface;

class Categorize {

    /**
     * Repository config.
     *
     * @var \Illuminate\Config\Repository
     */
    protected $config;

    /**
     * Category provider.
     *
     * @var \CategoryProviderInterface
     */
    protected $categoryProvider;

    /**
     * Category hierarchy provider.
     *
     * @var \CategoryHierarchyProviderInterface
     */
    protected $categoryHierarchyProvider;

    /**
     * Category relate provider.
     *
     * @var CategoryRelateProviderInterface
     */
    protected $categoryRelateProvider;

    /**
     * Construct.
     *
     * @param \Illuminate\Config\Repository       $config
     * @param \CategoryProviderInterface          $categoryProvider
     * @param \CategoryHierarchyProviderInterface $categoryHierarchyProvider
     * @param \CategoryRelateProviderInterface    $categoryRelateProvider
     */
    public function __construct(Repository $config,
                                CategoryProviderInterface $categoryProvider,
                                CategoryRelateProviderInterface $categoryRelateProvider,
                                CategoryHierarchyProviderInterface $categoryHierarchyProvider)

    {
        $this->config = $config;

        $this->categoryProvider = $categoryProvider;

        $this->categoryRelateProvider = $categoryRelateProvider;

        $this->categoryHierarchyProvider = $categoryHierarchyProvider;
    }

    /**
     * Change data array to object model.
     *
     * @param  array $data
     * @return object
     */
    public function prepare($data)
    {
        return $this->getCategoryProvider()->createModel()->fill($data);
    }

    /**
     * Get category provider.
     *
     * @return object
     */
    public function getCategoryProvider()
    {
        return $this->categoryProvider;
    }

    /**
     * Get category relate provider.
     *
     * @return object
     */
    public function getCategoryRelateProvider()
    {
        return $this->categoryRelateProvider;
    }

    /**
     * Get category hierarchy provider.
     *
     * @return object
     */
    public function getCategoryHierarchyProvider()
    {
        return $this->categoryHierarchyProvider;
    }

    /**
     * Buiding collections to tree.
     *
     * @param  Collection $source
     * @return object
     */
    public function tree(Collection $source)
    {
        $source->load(implode('.', array_fill(0, 20, 'children')));

        return $source;
    }

}


text:
import requests
import re

code = "035720"
re_enc = re.compile("encparam: '(.*)'", re.IGNORECASE)
re_id = re.compile("id: '([a-zA-Z0-9]*)' ?", re.IGNORECASE)

url = "https://companyinfo.stock.naver.com/v1/company/c1010001.aspx?cmp_cd={}".format(code)
html = requests.get(url).text
encparam = re_enc.search(html).group(1)
encid = re_id.search(html).group(1)

url = "https://companyinfo.stock.naver.com/v1/company/ajax/cF1001.aspx?cmp_cd={}&fin_typ=0&freq_typ=A&encparam={}&id={}".format(
    code, encparam, encid)
headers = {"Referer": "HACK"}
html = requests.get(url, headers=headers).text
print(html)


text:
package org.apache.derby.jdbc;

/**
   <P>
   This is a vacuous, deprecated class. At one time, it had real behavior and helped us support
   separate datasources for Java 5 and Java 6.
   Now that we no longer support Java 5, all functionality has migrated into the superclass, EmbeddedDataSource.
   This class is preserved for backward compatibility reasons.
   </P>
   @deprecated Use {@link EmbeddedDataSource} instead.
*/
public class EmbeddedDataSource40 extends EmbeddedDataSource
    implements javax.sql.DataSource /* compile-time check for 4.1 extension */
{
   private static final long serialVersionUID = 4472591890758954803L;

    public EmbeddedDataSource40() {
    }

}


text:
module Fog
  module AWS
    class STS
      class Real

        require 'fog/aws/parsers/sts/assume_role'

        # Assume Role
        #
        # ==== Parameters
        # * role_session_name<~String> - An identifier for the assumed role.
        # * role_arn<~String> - The ARN of the role the caller is assuming.
        # * external_id<~String> - An optional unique identifier required by the assuming role's trust identity.
        # * policy<~String> - An optional JSON policy document
        # * duration<~Integer> - Duration (of seconds) for the assumed role credentials to be valid (default 3600)
        #
        # ==== Returns
        # * response<~Excon::Response>:
        #   * body<~Hash>:
        #     * 'Arn'<~String>: The ARN of the assumed role/user
        #     * 'AccessKeyId'<~String>: The AWS access key of the temporary credentials for the assumed role
        #     * 'SecretAccessKey'<~String>: The AWS secret key of the temporary credentials for the assumed role
        #     * 'SessionToken'<~String>: The AWS session token of the temporary credentials for the assumed role
        #     * 'Expiration'<~Time>: The expiration time of the temporary credentials for the assumed role
        #
        # ==== See Also
        # http://docs.aws.amazon.com/STS/latest/APIReference/API_AssumeRole.html
        #

        def assume_role(role_session_name, role_arn, external_id=nil, policy=nil, duration=3600)
          request({
            'Action'          => 'AssumeRole',
            'RoleSessionName' => role_session_name,
            'RoleArn'         => role_arn,
            'Policy'          => policy && Fog::JSON.encode(policy),
            'DurationSeconds' => duration,
            'ExternalId'      => external_id,
            :idempotent       => true,
            :parser           => Fog::Parsers::AWS::STS::AssumeRole.new
          })
        end

      end
    end
  end
end


text:
<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN" "http://www.w3.org/TR/html4/loose.dtd">

<title>nl.esciencecenter.xenon (xenon-2.3.0 2.6.2 API for Xenon developers)</title>
<link rel="stylesheet" type="text/css" href="../../../stylesheet.css" title="Style">

nl.esciencecenter.xenon

Interfaces

AdaptorDescription

Classes

XenonPropertyDescription

Enums

XenonPropertyDescription.Type

Exceptions

InvalidCredentialException

InvalidLocationException

InvalidPropertyException

PropertyTypeException

UnknownAdaptorException

UnknownPropertyException

UnsupportedOperationException

XenonException

XenonRuntimeException


text:
package org.apache.gobblin.util.io;

import lombok.AllArgsConstructor;
import lombok.RequiredArgsConstructor;

import java.io.IOException;
import java.lang.reflect.GenericArrayType;
import java.lang.reflect.ParameterizedType;
import java.util.Collection;
import java.util.Map;

import org.apache.commons.lang3.ClassUtils;

import com.google.common.base.Optional;
import com.google.gson.Gson;
import com.google.gson.GsonBuilder;
import com.google.gson.JsonElement;
import com.google.gson.JsonObject;
import com.google.gson.JsonPrimitive;
import com.google.gson.TypeAdapter;
import com.google.gson.TypeAdapterFactory;
import com.google.gson.internal.Streams;
import com.google.gson.reflect.TypeToken;
import com.google.gson.stream.JsonReader;
import com.google.gson.stream.JsonWriter;

/**
 * A {@link Gson} interface adapter that makes it possible to serialize and deserialize polymorphic objects.
 *
 *

 *   This adapter will capture all instances of {@link #baseClass} and write them as
 *   {"object-type":"class.name", "object-data":"data"}, allowing for correct serialization and deserialization of
 *   polymorphic objects. The following types will not be captured by the adapter (i.e. they will be written by the
 *   default GSON writer):
 *   - Primitives and boxed primitives
 *   - Arrays
 *   - Collections
 *   - Maps
 *   Additionally, generic classes (e.g. class MyClass<T>) cannot be correctly decoded.
 *

 *
 *

 *   To use:
 *

 *          {@code
 *            MyClass object = new MyClass();
 *            Gson gson = GsonInterfaceAdapter.getGson(MyBaseClass.class);
 *            String json = gson.toJson(object);
 *            Myclass object2 = gson.fromJson(json, MyClass.class);
 *          }
 *

 *

 *
 *

 *   Note: a useful case is GsonInterfaceAdapter.getGson(Object.class), which will correctly serialize / deserialize
 *   all types except for java generics.
 *

 *
 * @param <T> The interface or abstract type to be serialized and deserialized with {@link Gson}.
 */
@RequiredArgsConstructor
public class GsonInterfaceAdapter implements TypeAdapterFactory {

  protected static final String OBJECT_TYPE = "object-type";
  protected static final String OBJECT_DATA = "object-data";

  private final Class<?> baseClass;

  @Override
  public <R> TypeAdapter<R> create(Gson gson, TypeToken<R> type) {
    if (ClassUtils.isPrimitiveOrWrapper(type.getRawType()) || type.getType() instanceof GenericArrayType
        || CharSequence.class.isAssignableFrom(type.getRawType())
        || (type.getType() instanceof ParameterizedType && (Collection.class.isAssignableFrom(type.getRawType())
            || Map.class.isAssignableFrom(type.getRawType())))) {
      // delegate primitives, arrays, collections, and maps
      return null;
    }
    if (!this.baseClass.isAssignableFrom(type.getRawType())) {
      // delegate anything not assignable from base class
      return null;
    }
    TypeAdapter<R> adapter = new InterfaceAdapter<>(gson, this, type);
    return adapter;
  }

  @AllArgsConstructor
  private static class InterfaceAdapter<R> extends TypeAdapter<R> {

    private final Gson gson;
    private final TypeAdapterFactory factory;
    private final TypeToken<R> typeToken;

    @Override
    public void write(JsonWriter out, R value) throws IOException {
      if (Optional.class.isAssignableFrom(this.typeToken.getRawType())) {
        Optional opt = (Optional) value;
        if (opt != null && opt.isPresent()) {
          Object actualValue = opt.get();
          writeObject(actualValue, out);
        } else {
          out.beginObject();
          out.endObject();
        }
      } else {
        writeObject(value, out);
      }
    }

    @Override
    public R read(JsonReader in) throws IOException {
      JsonElement element = Streams.parse(in);
      if (element.isJsonNull()) {
        return readNull();
      }
      JsonObject jsonObject = element.getAsJsonObject();

      if (this.typeToken.getRawType() == Optional.class) {
        if (jsonObject.has(OBJECT_TYPE)) {
          return (R) Optional.of(readValue(jsonObject, null));
        } else if (jsonObject.entrySet().isEmpty()) {
          return (R) Optional.absent();
        } else {
          throw new IOException("No class found for Optional value.");
        }
      }
      return this.readValue(jsonObject, this.typeToken);
    }

    private <S> S readNull() {
      if (this.typeToken.getRawType() == Optional.class) {
        return (S) Optional.absent();
      }
      return null;
    }

    private <S> void writeObject(S value, JsonWriter out) throws IOException {
      if (value != null) {
        JsonObject jsonObject = new JsonObject();
        jsonObject.add(OBJECT_TYPE, new JsonPrimitive(value.getClass().getName()));
        TypeAdapter<S> delegate =
            (TypeAdapter<S>) this.gson.getDelegateAdapter(this.factory, TypeToken.get(value.getClass()));
        jsonObject.add(OBJECT_DATA, delegate.toJsonTree(value));
        Streams.write(jsonObject, out);
      } else {
        out.nullValue();
      }
    }

    private <S> S readValue(JsonObject jsonObject, TypeToken<S> defaultTypeToken) throws IOException {
      try {
        TypeToken<S> actualTypeToken;
        if (jsonObject.isJsonNull()) {
          return null;
        } else if (jsonObject.has(OBJECT_TYPE)) {
          String className = jsonObject.get(OBJECT_TYPE).getAsString();
          Class<S> klazz = (Class<S>) Class.forName(className);
          actualTypeToken = TypeToken.get(klazz);
        } else if (defaultTypeToken != null) {
          actualTypeToken = defaultTypeToken;
        } else {
          throw new IOException("Could not determine TypeToken.");
        }
        TypeAdapter<S> delegate = this.gson.getDelegateAdapter(this.factory, actualTypeToken);
        S value = delegate.fromJsonTree(jsonObject.get(OBJECT_DATA));
        return value;
      } catch (ClassNotFoundException cnfe) {
        throw new IOException(cnfe);
      }
    }

  }

  public static <T> Gson getGson(Class<T> clazz) {
    Gson gson = new GsonBuilder().registerTypeAdapterFactory(new GsonInterfaceAdapter(clazz)).create();
    return gson;
  }
}


text:
package org.res4j.dao.xml.xinclude;

import java.math.BigDecimal;
import java.net.URL;
import java.util.Locale;

import javax.xml.bind.annotation.XmlTransient;

import org.junit.Test;
import org.res4j.dao.xml.XmlDaoConfig;
import org.res4j.test.AbstractElTest;

import test.bean.jaxb.nonamespace.Catalog;
import test.bean.jaxb.nonamespace.Product;
import test.bean.jaxb.nonamespace.ProductBundle;

/**
 * Test for XInclude using {@link XmlDaoConfig}.
 */
@XmlTransient
public class XIncludeTest extends AbstractElTest {

	// Implementation note: an XInclude relative URL can be computed only if InputSource.setSystemId(String) is called.

	@Test
	public void mapsAndSimpleTypes() throws Exception {
		getContent();
		assertExpression(3, "content.size()");
		assertExpression(4, "content.map_1.size()");
		assertExpression("String 1.1", "content.map_1.string_1_1");
		// map_1_2 file replaces the include tag and a "base" attribute is added
		// to the map_1_2 element.
		assertExpression(5, "content.map_1.map_1_2.size()");
		assertExpression("map_1_2", "content.map_1.map_1_2.id");
		assertExpression("customType", "content.map_1.map_1_2.type");
		assertExpression("XIncludeTest.mapsAndSimpleTypes.map_1_2.xml", "content.map_1.map_1_2.base");
		assertExpression(Boolean.TRUE, "content.map_1.map_1_2.boolean_1_2_1");
		assertExpression(Byte.valueOf((byte) 123), "content.map_1.map_1_2.byte_1_2_2");
		assertExpression(3, "content.map_1.map_1_4.size()");
		assertExpression("map_1_4", "content.map_1.map_1_4.id");
		assertExpression("String 1.4.1", "content.map_1.map_1_4.string_1_4_1");
		assertExpression("String 1.4.2", "content.map_1.map_1_4.string_1_4_2");
		assertExpression(Double.valueOf("-1234567890.123e999"), "content.double_2");
		// map_3 file replaces the include tag and a "base" attribute is added
		// to the map_3 element.
		assertExpression(3, "content.map_3.size()");
		assertExpression("XIncludeTest.mapsAndSimpleTypes.map_3.xml", "content.map_3.base");
		assertExpression(Long.valueOf(-123456789012345678L), "content.map_3.long_3_1");
		assertExpression(new URL("http://www.java.com/en/about/"), "content.map_3.url_3_2");
	}

	@Test
	public void jaxb() throws Exception {
		getContent(XML_SUFFIX, Locale.CANADA);
		assertInstanceOf(Catalog.class, content);
		assertExpression(3, "content.products.size()");
		assertExpression("Big Jack", "content.products[0].name");
		assertExpression("French Fries - Large", "content.products[1].name");
		assertExpression(1, "content.bundles.size()");
		assertExpression("bigJackTrio", "content.bundles[0].id");
		assertExpression("Big Jack Trio", "content.bundles[0].name");
		assertExpression(3, "content.bundles[0].subProducts.size()");
		assertExpression("bigJack", "content.bundles[0].subProducts[0].id");
		assertExpression("largeFries", "content.bundles[0].subProducts[1].id");
		assertExpression("largeCola", "content.bundles[0].subProducts[2].id");
		assertExpression(new BigDecimal("6.20"), "content.bundles[0].price");
		assertExpression(new BigDecimal("7.15"), "content.bundles[0].regularPrice");
		assertExpression(new BigDecimal("0.95"), "content.bundles[0].discount");
		getContent(XML_SUFFIX, Locale.CANADA_FRENCH);
		assertExpression("Frites - grand format", "content.products[1].name");
		assertExpression("Trio Big Jack", "content.bundles[0].name");
		getContent(XML_SUFFIX, Locale.US);
		assertExpression("Liberty Fries - Large", "content.products[1].name");
		assertExpression(new BigDecimal("5.95"), "content.bundles[0].price");
		assertExpression(new BigDecimal("6.90"), "content.bundles[0].regularPrice");
		assertExpression(new BigDecimal("0.95"), "content.bundles[0].discount");
	}

	@Test
	public void mapsAndJaxb() throws Exception {
		getContent();
		assertExpression(3, "content.size()");
		assertExpression(4, "content.map_1.size()");
		assertExpressionInstanceOf(Product.class, "content.map_1.map_1_2.product_1_2_2");
		assertExpression("bigJack", "content.map_1.map_1_2.product_1_2_2.id");
		assertExpression("Big Jack", "content.map_1.map_1_2.product_1_2_2.name");
		assertExpressionInstanceOf(ProductBundle.class, "content.map_1.map_1_2.bundle_1_2_5");
		assertExpression("bigJackTrio", "content.map_1.map_1_2.bundle_1_2_5.id");
		assertExpression("Big Jack Trio", "content.map_1.map_1_2.bundle_1_2_5.name");
		// Note: Hybrid SAX/JAXB supports @XmlIDREF.
		assertExpression(3, "content.map_1.map_1_2.bundle_1_2_5.subProducts.size()");
		assertExpression(new BigDecimal("5.95"), "content.map_1.map_1_2.bundle_1_2_5.price");
		// Note: Java beans may have computed fields as the following.
		assertExpression(new BigDecimal("6.90"), "content.map_1.map_1_2.bundle_1_2_5.regularPrice");
		assertExpression(new BigDecimal("0.95"), "content.map_1.map_1_2.bundle_1_2_5.discount");
	}

}


text:
<Record>
<Term>Histamine H3 Antagonists</Term>
<SemanticType>Pharmacologic Substance</SemanticType>
<ParentTerm>Histamine Antagonists</ParentTerm>
<ClassificationPath>Chemical Actions and Uses/Pharmacologic Actions/Molecular Mechanisms of Pharmacological Action/Neurotransmitter Agents/Histamine Agents/Histamine Antagonists/Histamine H3 Antagonists</ClassificationPath>
<ClassificationPath>Chemical Actions and Uses/Pharmacologic Actions/Physiological Effects of Drugs/Neurotransmitter Agents/Histamine Agents/Histamine Antagonists/Histamine H3 Antagonists</ClassificationPath>
<BroaderTerm>Pharmacologic Actions</BroaderTerm>
<BroaderTerm>Histamine Agents</BroaderTerm>
<BroaderTerm>Histamine Antagonists</BroaderTerm>
<BroaderTerm>Chemical Actions and Uses</BroaderTerm>
<BroaderTerm>Histamine H3 Antagonists</BroaderTerm>
<BroaderTerm>Molecular Mechanisms of Pharmacological Action</BroaderTerm>
<BroaderTerm>Physiological Effects of Drugs</BroaderTerm>
<BroaderTerm>Neurotransmitter Agents</BroaderTerm>
<Synonym>Histamine H3 Receptor Antagonists</Synonym>
<Synonym>H3 Antihistaminics</Synonym>
<Synonym>Histamine H3 Antagonists</Synonym>
<Synonym>Histamine H3 Blockers</Synonym>
<Description>Drugs that selectively bind to but do not activate HISTAMINE H3 RECEPTORS. They have been used to correct SLEEP DISORDERS and MEMORY DISORDERS.</Description>
<Source>MeSH</Source>
</Record>


text:
<?php

namespace Zotlabs\Module\Admin;

use \Zotlabs\Storage\GitRepo as GitRepo;
use \Michelf\MarkdownExtra;

class Plugins {

	function post() {

		if(argc() > 2 && is_file("addon/" . argv(2) . "/" . argv(2) . ".php")) {
			@include_once("addon/" . argv(2) . "/" . argv(2) . ".php");
			if(function_exists(argv(2).'_plugin_admin_post')) {
				$func = argv(2) . '_plugin_admin_post';
				$func($a);
			}

			goaway(z_root() . '/admin/plugins/' . argv(2) );
		}
		elseif(argc() > 2) {
			switch(argv(2)) {
				case 'updaterepo':
					if (array_key_exists('repoName', $_REQUEST)) {
						$repoName = $_REQUEST['repoName'];
					}
					else {
						json_return_and_die(array('message' => 'No repo name provided.', 'success' => false));
					}
					$extendDir = 'store/[data]/git/sys/extend';
					$addonDir = $extendDir . '/addon';
					if (!file_exists($extendDir)) {
						if (!mkdir($extendDir, 0770, true)) {
							logger('Error creating extend folder: ' . $extendDir);
							json_return_and_die(array('message' => 'Error creating extend folder: ' . $extendDir, 'success' => false));
						}
						else {
							if (!symlink(realpath('extend/addon'), $addonDir)) {
								logger('Error creating symlink to addon folder: ' . $addonDir);
								json_return_and_die(array('message' => 'Error creating symlink to addon folder: ' . $addonDir, 'success' => false));
							}
						}
					}
					$repoDir = 'store/[data]/git/sys/extend/addon/' . $repoName;
					if (!is_dir($repoDir)) {
						logger('Repo directory does not exist: ' . $repoDir);
						json_return_and_die(array('message' => 'Invalid addon repo.', 'success' => false));
					}
					if (!is_writable($repoDir)) {
						logger('Repo directory not writable to web server: ' . $repoDir);
						json_return_and_die(array('message' => 'Repo directory not writable to web server.', 'success' => false));
					}
					$git = new GitRepo('sys', null, false, $repoName, $repoDir);
					try {
						if ($git->pull()) {
							$files = array_diff(scandir($repoDir), array('.', '..'));
							foreach ($files as $file) {
								if (is_dir($repoDir . '/' . $file) && $file !== '.git') {
									$source = '../extend/addon/' . $repoName . '/' . $file;
									$target = realpath('addon/') . '/' . $file;
									unlink($target);
									if (!symlink($source, $target)) {
										logger('Error linking addons to /addon');
										json_return_and_die(array('message' => 'Error linking addons to /addon', 'success' => false));
									}
								}
							}
							json_return_and_die(array('message' => 'Repo updated.', 'success' => true));
						} else {
							json_return_and_die(array('message' => 'Error updating addon repo.', 'success' => false));
						}
					} catch (\PHPGit\Exception\GitException $e) {
						json_return_and_die(array('message' => 'Error updating addon repo.', 'success' => false));
					}
				case 'removerepo':
					if (array_key_exists('repoName', $_REQUEST)) {
						$repoName = $_REQUEST['repoName'];
					} else {
						json_return_and_die(array('message' => 'No repo name provided.', 'success' => false));
					}
					$extendDir = 'store/[data]/git/sys/extend';
					$addonDir = $extendDir . '/addon';
					if (!file_exists($extendDir)) {
						if (!mkdir($extendDir, 0770, true)) {
							logger('Error creating extend folder: ' . $extendDir);
							json_return_and_die(array('message' => 'Error creating extend folder: ' . $extendDir, 'success' => false));
						} else {
							if (!symlink(realpath('extend/addon'), $addonDir)) {
								logger('Error creating symlink to addon folder: ' . $addonDir);
								json_return_and_die(array('message' => 'Error creating symlink to addon folder: ' . $addonDir, 'success' => false));
							}
						}
					}
					$repoDir = 'store/[data]/git/sys/extend/addon/' . $repoName;
					if (!is_dir($repoDir)) {
						logger('Repo directory does not exist: ' . $repoDir);
						json_return_and_die(array('message' => 'Invalid addon repo.', 'success' => false));
					}
					if (!is_writable($repoDir)) {
						logger('Repo directory not writable to web server: ' . $repoDir);
						json_return_and_die(array('message' => 'Repo directory not writable to web server.', 'success' => false));
					}
					/// @TODO remove directory and unlink /addon/files
					if (rrmdir($repoDir)) {
						json_return_and_die(array('message' => 'Repo deleted.', 'success' => true));
					} else {
						json_return_and_die(array('message' => 'Error deleting addon repo.', 'success' => false));
					}
				case 'installrepo':
					if (array_key_exists('repoURL', $_REQUEST)) {
						require_once('library/PHPGit.autoload.php');
						$repoURL = $_REQUEST['repoURL'];
						$extendDir = 'store/[data]/git/sys/extend';
						$addonDir = $extendDir . '/addon';
						if (!file_exists($extendDir)) {
							if (!mkdir($extendDir, 0770, true)) {
								logger('Error creating extend folder: ' . $extendDir);
								json_return_and_die(array('message' => 'Error creating extend folder: ' . $extendDir, 'success' => false));
							} else {
								if (!symlink(realpath('extend/addon'), $addonDir)) {
									logger('Error creating symlink to addon folder: ' . $addonDir);
									json_return_and_die(array('message' => 'Error creating symlink to addon folder: ' . $addonDir, 'success' => false));
								}
							}
						}
						if (!is_writable($extendDir)) {
							logger('Directory not writable to web server: ' . $extendDir);
							json_return_and_die(array('message' => 'Directory not writable to web server.', 'success' => false));
						}
						$repoName = null;
						if (array_key_exists('repoName', $_REQUEST) && $_REQUEST['repoName'] !== '') {
							$repoName = $_REQUEST['repoName'];
						} else {
							$repoName = GitRepo::getRepoNameFromURL($repoURL);
						}
						if (!$repoName) {
							logger('Invalid git repo');
							json_return_and_die(array('message' => 'Invalid git repo', 'success' => false));
						}
						$repoDir = $addonDir . '/' . $repoName;
						$tempRepoBaseDir = 'store/[data]/git/sys/temp/';
						$tempAddonDir = $tempRepoBaseDir . $repoName;

						if (!is_writable($addonDir) || !is_writable($tempAddonDir)) {
							logger('Temp repo directory or /extend/addon not writable to web server: ' . $tempAddonDir);
							json_return_and_die(array('message' => 'Temp repo directory not writable to web server.', 'success' => false));
						}
						rename($tempAddonDir, $repoDir);

						if (!is_writable(realpath('addon/'))) {
							logger('/addon directory not writable to web server: ' . $tempAddonDir);
							json_return_and_die(array('message' => '/addon directory not writable to web server.', 'success' => false));
						}
						$files = array_diff(scandir($repoDir), array('.', '..'));
						foreach ($files as $file) {
							if (is_dir($repoDir . '/' . $file) && $file !== '.git') {
								$source = '../extend/addon/' . $repoName . '/' . $file;
								$target = realpath('addon/') . '/' . $file;
								unlink($target);
								if (!symlink($source, $target)) {
									logger('Error linking addons to /addon');
									json_return_and_die(array('message' => 'Error linking addons to /addon', 'success' => false));
								}
							}
						}
						$git = new GitRepo('sys', $repoURL, false, $repoName, $repoDir);
						$repo = $git->probeRepo();
						json_return_and_die(array('repo' => $repo, 'message' => '', 'success' => true));
					}
				case 'addrepo':
					if (array_key_exists('repoURL', $_REQUEST)) {
						require_once('library/PHPGit.autoload.php');
						$repoURL = $_REQUEST['repoURL'];
						$extendDir = 'store/[data]/git/sys/extend';
						$addonDir = $extendDir . '/addon';
						$tempAddonDir = realpath('store/[data]') . '/git/sys/temp';
						if (!file_exists($extendDir)) {
							if (!mkdir($extendDir, 0770, true)) {
								logger('Error creating extend folder: ' . $extendDir);
								json_return_and_die(array('message' => 'Error creating extend folder: ' . $extendDir, 'success' => false));
							} else {
								if (!symlink(realpath('extend/addon'), $addonDir)) {
									logger('Error creating symlink to addon folder: ' . $addonDir);
									json_return_and_die(array('message' => 'Error creating symlink to addon folder: ' . $addonDir, 'success' => false));
								}
							}
						}
						if (!is_dir($tempAddonDir)) {
							if (!mkdir($tempAddonDir, 0770, true)) {
								logger('Error creating temp plugin repo folder: ' . $tempAddonDir);
								json_return_and_die(array('message' => 'Error creating temp plugin repo folder: ' . $tempAddonDir, 'success' => false));
							}
						}
						$repoName = null;
						if (array_key_exists('repoName', $_REQUEST) && $_REQUEST['repoName'] !== '') {
							$repoName = $_REQUEST['repoName'];
						} else {
							$repoName = GitRepo::getRepoNameFromURL($repoURL);
						}
						if (!$repoName) {
							logger('Invalid git repo');
							json_return_and_die(array('message' => 'Invalid git repo: ' . $repoName, 'success' => false));
						}
						$repoDir = $tempAddonDir . '/' . $repoName;
						if (!is_writable($tempAddonDir)) {
							logger('Temporary directory for new addon repo is not writable to web server: ' . $tempAddonDir);
							json_return_and_die(array('message' => 'Temporary directory for new addon repo is not writable to web server.', 'success' => false));
						}

						$git = new GitRepo('sys', $repoURL, true, $repoName, $repoDir);

						$remotes = $git->git->remote();
						$fetchURL = $remotes['origin']['fetch'];
						if ($fetchURL !== $git->url) {
							if (rrmdir($repoDir)) {
								$git = new GitRepo('sys', $repoURL, true, $repoName, $repoDir);
							} else {
								json_return_and_die(array('message' => 'Error deleting existing addon repo.', 'success' => false));
							}
						}
						$repo = $git->probeRepo();
						$repo['readme'] = $repo['manifest'] = null;
						foreach ($git->git->tree('master') as $object) {
							if ($object['type'] == 'blob' && (strtolower($object['file']) === 'readme.md' || strtolower($object['file']) === 'readme')) {
								$repo['readme'] = MarkdownExtra::defaultTransform($git->git->cat->blob($object['hash']));
							} else if ($object['type'] == 'blob' && strtolower($object['file']) === 'manifest.json') {
								$repo['manifest'] = $git->git->cat->blob($object['hash']);
							}
						}
						json_return_and_die(array('repo' => $repo, 'message' => '', 'success' => true));
					} else {
						json_return_and_die(array('message' => 'No repo URL provided', 'success' => false));
					}
					break;
				default:
					break;
			}
		}
	}

	function get() {

		if (\App::$argc == 3){
			$plugin = \App::$argv[2];
			if (!is_file("addon/$plugin/$plugin.php")){
				notice( t("Item not found.") );
				return '';
			}

			$enabled = in_array($plugin,\App::$plugins);
			$info = get_plugin_info($plugin);
			$x = check_plugin_versions($info);

			if($enabled && ! $x) {
				$enabled = false;
				$idz = array_search($plugin, \App::$plugins);
				if ($idz !== false) {
					unset(\App::$plugins[$idz]);
					uninstall_plugin($plugin);
					set_config("system","addon", implode(", ",\App::$plugins));
				}
			}
			$info['disabled'] = 1-intval($x);

			if (x($_GET,"a") && $_GET['a']=="t"){
				check_form_security_token_redirectOnErr('/admin/plugins', 'admin_plugins', 't');
				$pinstalled = false;
				// Toggle plugin status
				$idx = array_search($plugin, \App::$plugins);
				if ($idx !== false){
					unset(\App::$plugins[$idx]);
					uninstall_plugin($plugin);
					$pinstalled = false;
					info( sprintf( t("Plugin %s disabled."), $plugin ) );
				} else {
					\App::$plugins[] = $plugin;
					install_plugin($plugin);
					$pinstalled = true;
					info( sprintf( t("Plugin %s enabled."), $plugin ) );
				}
				set_config("system","addon", implode(", ",\App::$plugins));

				if($pinstalled) {
					@require_once("addon/$plugin/$plugin.php");
					if(function_exists($plugin.'_plugin_admin'))
						goaway(z_root() . '/admin/plugins/' . $plugin);
				}
				goaway(z_root() . '/admin/plugins' );
			}

			if (in_array($plugin, \App::$plugins)){
				$status = 'on';
				$action = t('Disable');
			} else {
				$status = 'off';
				$action =  t('Enable');
			}

			$readme = null;
			if (is_file("addon/$plugin/README.md")){
				$readme = file_get_contents("addon/$plugin/README.md");
				$readme = MarkdownExtra::defaultTransform($readme);
			} else if (is_file("addon/$plugin/README")){
				$readme = "<pre>". file_get_contents("addon/$plugin/README") ."</pre>";
			}

			$admin_form = '';

			$r = q("select * from addon where plugin_admin = 1 and aname = '%s' limit 1",
				dbesc($plugin)
			);

			if($r) {
				@require_once("addon/$plugin/$plugin.php");
				if(function_exists($plugin.'_plugin_admin')) {
					$func = $plugin.'_plugin_admin';
					$func($a, $admin_form);
				}
			}

			$t = get_markup_template('admin_plugins_details.tpl');
			return replace_macros($t, array(
				'$title' => t('Administration'),
				'$page' => t('Plugins'),
				'$toggle' => t('Toggle'),
				'$settings' => t('Settings'),
				'$baseurl' => z_root(),

				'$plugin' => $plugin,
				'$status' => $status,
				'$action' => $action,
				'$info' => $info,
				'$str_author' => t('Author: '),
				'$str_maintainer' => t('Maintainer: '),
				'$str_minversion' => t('Minimum project version: '),
				'$str_maxversion' => t('Maximum project version: '),
				'$str_minphpversion' => t('Minimum PHP version: '),
				'$str_serverroles' => t('Compatible Server Roles: '),
				'$str_requires' => t('Requires: '),
				'$disabled' => t('Disabled - version incompatibility'),

				'$admin_form' => $admin_form,
				'$function' => 'plugins',
				'$screenshot' => '',
				'$readme' => $readme,

				'$form_security_token' => get_form_security_token('admin_plugins'),
			));
		}

		$plugins = array();
		$files = glob('addon/*/');
		if($files) {
			foreach($files as $file) {
				if (is_dir($file)){
					list($tmp, $id) = array_map('trim', explode('/', $file));
					$info = get_plugin_info($id);
					$enabled = in_array($id,\App::$plugins);
					$x = check_plugin_versions($info);

					if($enabled && ! $x) {
						$enabled = false;
						$idz = array_search($id, \App::$plugins);
						if ($idz !== false) {
							unset(\App::$plugins[$idz]);
							uninstall_plugin($id);
							set_config("system","addon", implode(", ",\App::$plugins));
						}
					}
					$info['disabled'] = 1-intval($x);

					$plugins[] = array( $id, (($enabled)?"on":"off") , $info);
				}
			}
		}

		usort($plugins,'self::plugin_sort');

		$allowManageRepos = false;
		if(is_writable('extend/addon') && is_writable('store/[data]')) {
			$allowManageRepos = true;
		}

		$admin_plugins_add_repo_form= replace_macros(
			get_markup_template('admin_plugins_addrepo.tpl'), array(
				'$post' => 'admin/plugins/addrepo',
				'$desc' => t('Enter the public git repository URL of the plugin repo.'),
				'$repoURL' => array('repoURL', t('Plugin repo git URL'), '', ''),
				'$repoName' => array('repoName', t('Custom repo name'), '', '', t('(optional)')),
				'$submit' => t('Download Plugin Repo')
			)
		);
		$newRepoModalID = random_string(3);
		$newRepoModal = replace_macros(
			get_markup_template('generic_modal.tpl'), array(
				'$id' => $newRepoModalID,
				'$title' => t('Install new repo'),
				'$ok' => t('Install'),
				'$cancel' => t('Cancel')
			)
		);

		$reponames = $this->listAddonRepos();
		$addonrepos = [];
		foreach($reponames as $repo) {
			$addonrepos[] = array('name' => $repo, 'description' => '');
			/// @TODO Parse repo info
		}

		$t = get_markup_template('admin_plugins.tpl');
		return replace_macros($t, array(
			'$title' => t('Administration'),
			'$page' => t('Plugins'),
			'$submit' => t('Submit'),
			'$baseurl' => z_root(),
			'$function' => 'plugins',
			'$plugins' => $plugins,
			'$disabled' => t('Disabled - version incompatibility'),
			'$form_security_token' => get_form_security_token('admin_plugins'),
			'$allowManageRepos' => $allowManageRepos,
			'$managerepos' => t('Manage Repos'),
			'$installedtitle' => t('Installed Plugin Repositories'),
			'$addnewrepotitle' =>	t('Install a New Plugin Repository'),
			'$expandform' => false,
			'$form' => $admin_plugins_add_repo_form,
			'$newRepoModal' => $newRepoModal,
			'$newRepoModalID' => $newRepoModalID,
			'$addonrepos' => $addonrepos,
			'$repoUpdateButton' => t('Update'),
			'$repoBranchButton' => t('Switch branch'),
			'$repoRemoveButton' => t('Remove')
		));
	}

	function listAddonRepos() {
		$addonrepos = [];
		$addonDir = 'extend/addon/';
		if(is_dir($addonDir)) {
			if ($handle = opendir($addonDir)) {
				while (false !== ($entry = readdir($handle))) {
					if ($entry != "." && $entry != "..") {
						$addonrepos[] = $entry;
					}
				}
				closedir($handle);
			}
		}
		return $addonrepos;
	}

	static public function plugin_sort($a,$b) {
		return(strcmp(strtolower($a[2]['name']),strtolower($b[2]['name'])));
	}

}