package de.fhg.iais.cortex.im.worker;

import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.OutputStreamWriter;
import java.nio.file.FileVisitResult;
import java.nio.file.Path;
import java.nio.file.SimpleFileVisitor;
import java.nio.file.attribute.BasicFileAttributes;
import java.text.DateFormat;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Calendar;
import java.util.Collections;
import java.util.Date;
import java.util.GregorianCalendar;
import java.util.HashMap;
import java.util.LinkedList;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import javax.inject.Inject;
import javax.inject.Named;

import org.apache.commons.io.IOUtils;
import org.apache.commons.lang.StringUtils;
import org.json.JSONArray;
import org.json.JSONException;
import org.json.JSONObject;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import com.google.common.base.Strings;

import de.fhg.iais.commons.Files;
import de.fhg.iais.commons.dbc.Check;
import de.fhg.iais.commons.dbc.DbcException;
import de.fhg.iais.cortex.im.ComparisonResult;
import de.fhg.iais.cortex.im.ListComparator;
import de.fhg.iais.cortex.im.bookkeeping.BookkeepingData;
import de.fhg.iais.cortex.im.bookkeeping.BookkeepingManager;
import de.fhg.iais.cortex.im.bookkeeping.BookkeepingWrapper;
import de.fhg.iais.cortex.im.bookkeeping.BookkeepingWrapperFactory;
import de.fhg.iais.cortex.im.server.IngestMultiplexerServer;
import de.fhg.iais.cortex.im.utils.ProductionLine;
import de.fhg.iais.cortex.im.utils.Result;
import de.fhg.iais.cortex.report.common.AscState;
import de.fhg.iais.cortex.report.common.IngestState;

/**
 * This class is responsible to control the file system persisting all data for the {@link IngestMultiplexerServer}.
 * One and only one object may be created by the {@link IngestMultiplexerServer}, one and only one object exists for
 * <b>all</b> production lines. As well as the {@link IngestMultiplexerServer} this is a <b>single point of
 * failure</b>. Change it carefully ... .<br>
 * <br>
 * The file system controlled by this class persists <b>all data of all ingests of the whole lifetime of a
 * digital library.</b>
 */
public final class SipDirectoryManager {
    private static final Logger LOG = LoggerFactory.getLogger(SipDirectoryManager.class);
    public static final boolean EXCESSIVE_DEBUG = true;
    private static final Pattern PATH_PART = Pattern.compile("[\\w-,]+");

    private static final boolean IS_WINDOWS = StringUtils.containsIgnoreCase(System.getProperty("os.name"), "win");

    private static final ThreadLocal<DateFormat> COMPARISON_DATE_FORMAT = new ThreadLocal<DateFormat>() {
        @Override
        protected DateFormat initialValue() {
            SimpleDateFormat format = new SimpleDateFormat("yyyyMMddHHmmss");
            return format;
        };
    };;

    public static final String DIR_SIP = "sip";

    public static final String TRANSFORMED_IDS_FILE = "ascTransformed.zip";
    public static final String DIR_BOOKKEEPING = "bookkeeping";
    public static final String PATH_FAILED_IDS_TREFIX = "failed_";

    private final File dataPath;

    private final File comparisonPath;
    private final BookkeepingManager bkManager;
    private final BookkeepingWrapperFactory factory;

    /**
     * create a new {@link SipDirectoryManager}, that is responsible to control the file system persisting all
     * data for the {@link IngestMultiplexerServer}. This constructor is used once and only once by the {@link IngestMultiplexerServer}.
     * 
     * @param dataPath the directory in which all data ingests ever done should be stored
     * @param comparisonPath the directory in which comparisons between ingests should be stored
     */
    @Inject
    public SipDirectoryManager(
        @Named("cs.ingest.multiplexer.storage") String sipDirectoryManagerDataPath,
        @Named("cs.ingest.multiplexer.comparisons") String sipDirectoryManagerComparisonPath,
        BookkeepingWrapperFactory factory,
        BookkeepingManager bkManager) {
        Check.notNull(sipDirectoryManagerDataPath);
        this.dataPath = new File(sipDirectoryManagerDataPath);
        this.comparisonPath = new File(sipDirectoryManagerComparisonPath);
        this.bkManager = bkManager;
        this.factory = factory;

        if ( this.dataPath.isDirectory() ) { //NOSONAR
            // OK
        } else if ( !this.dataPath.exists() ) {
            this.dataPath.mkdirs();
        } else {
            throw new DbcException("Invalid data path " + this.dataPath);
        }
        if ( this.comparisonPath.isDirectory() ) { //NOSONAR
            // OK
        } else if ( !this.comparisonPath.exists() ) {
            this.comparisonPath.mkdirs();
        } else {
            throw new DbcException("Invalid comparison " + this.comparisonPath);
        }

    }

    /**
     * create the directory structure needed for a fresh ingest, i.e. a new revision. Directories are created
     * if needed. The creation succeeds only if all previous revisions of this ingest have terminated or have
     * been aborted.<br>
     * <br>
     * <b>The returned {@link SipWriter} object is <i>unique</i> for this revision. It is thread-safe, i.e. it
     * may be distributed to many SIP-generating threads</b>
     * 
     * @param providerId not null
     * @param ingestId not null
     * @param revisionId not null
     * @param line maybe null. In this case the bookkeeping files for productive systems are not
     *        generated. They will be generated when a {@link SipReader}
     * @return a {@link SipWriter} object.This is the <b>only</b> object, that may be used to store SIPs into
     *         this revision.
     */
    public Result<SipWriter> createSipWriter(
        final String providerId,
        final String ingestId,
        final String revisionId,
        String providerDescription,
        List<ProductionLine> productionLines) {

        try {
            checkSipNames(providerId);
            checkSipNames(ingestId);
            checkSipNames(revisionId);

            File provider = makeChildDirectoryAvailable(this.dataPath, providerId, true);
            File ingest = makeChildDirectoryAvailable(provider, ingestId, true);

            String haveAllRevisionsTerminated = haveAllRevisionsTerminated(ingest, providerId, revisionId, productionLines);
            if ( haveAllRevisionsTerminated != null ) {
                return Result.makeError(haveAllRevisionsTerminated);
            }

            File revision = makeChildDirectoryAvailable(ingest, revisionId, true);

            makeChildDirectoryAvailable(revision, DIR_SIP, false);
            makeChildDirectoryAvailable(revision, DIR_BOOKKEEPING, false);

            List<BookkeepingWrapper> wrappers = new ArrayList<>();
            for ( ProductionLine line : productionLines ) {
                String name = line.getName();
                BookkeepingWrapper wrapper = this.factory.create(providerId, ingestId, revisionId, name, providerDescription);
                wrapper.setAscState(AscState.REVISION_IN_CONSTRUCTION);
                wrapper.setCurrent(0L);
                wrapper.setIngested("");
                wrapper.setIngestState(IngestState.READY_TO_START);
                wrapper.setAscTotal(0);
                wrapper.setErrorCount(0);
                wrapper.updateInDB();
                wrappers.add(wrapper);
            }

            return Result.makeOk(new SipWriter(wrappers, revision));
        } catch ( Exception e ) {
            String msg = "Problems the with underlying file system - SIPWriter cannot be created";
            LOG.error(msg, e);
            return Result.makeError(msg);
        }
    }

    /**
     * get a revision. May be called only by the master thread of the {@link IngestMultiplexerServer}, and for
     * analysing the revision data.
     * 
     * @return the revision as a SIP reader
     */
    public SipReader getRevision(final String providerId, final String ingestId, final String revisionId, List<ProductionLine> productiveSystems) {
        checkSipNames(providerId);
        checkSipNames(ingestId);
        checkSipNames(revisionId);
        Check.notNull(productiveSystems);

        File provider = checkChildDirectory(this.dataPath, providerId, "provider directory does not exist");
        File ingest = checkChildDirectory(provider, ingestId, "ingest directory does not exist");
        File revisionDir = checkChildDirectory(ingest, revisionId, "revision directory does not exist");

        Map<String, BookkeepingWrapper> bookkeepingData = getBookkeepingForAllProdLines(providerId, ingestId, revisionId, productiveSystems);

        SipReader sipReader = new SipReader(bookkeepingData, revisionDir, providerId, ingestId, revisionId);

        return sipReader;
    }

    /**
     * return ingests data concerning the ingest specified by the parameters.
     * 
     * @param providerId one of them or all of them if the parameter is null
     * @param ingestId one of them or all of them if the parameter is null
     * @param revisionId one of them or all of them if the parameter is null
     * @param indent if > 0 the result is pretty-printed and indent is taken as number of indenting each level
     * @return a String describing the state of ingests specified by providerId, ingestId and revisionId.
     */
    public String getIngestData(String providerId, String ingestId, String revisionId, int indent) {
        try {
            JSONObject resultJson = new JSONObject();
            List<BookkeepingData> all = this.bkManager.getAllBookkeepingData();
            if ( providerId == null ) {
                for ( BookkeepingData data : all ) {
                    resultJson.append("result", data.getProviderId());
                }
            } else {
                List<BookkeepingData> providers = this.bkManager.getBookkeepingDataByProvider(providerId);
                if ( ingestId == null ) {
                    for ( BookkeepingData data : providers ) {
                        resultJson.append("result", data.getIngestId());
                    }
                } else {
                    List<BookkeepingData> ingests = this.bkManager.getBookkeepingDataByIngest(providerId, ingestId);
                    if ( revisionId == null ) {
                        for ( BookkeepingData data : ingests ) {
                            resultJson.append("result", data.getRevisionId());
                        }
                    } else {
                        List<BookkeepingData> revision = this.bkManager.getBookkeepingDataByRevision(providerId, ingestId, revisionId);
                        getRevisionInfo(resultJson, revision);
                    }
                }
            }
            return indent > 0 ? resultJson.toString(indent) : resultJson.toString();
        } catch ( JSONException e ) {
            LOG.error("JSON exception", e);
            return "";
        }
    }

    /**
     * Return a list of revisions (as SipReader) whose <b>ingest</b> has not yet finished successfully.<br>
     * If a revision is found, whose SIPs are not completely <b>generated</b> by the ASC, this is considered
     * an aborted revision. This case arises if the ASC has been terminated by a kill-command or the power
     * switch has been turned off when the ASC was running, for example. An aborted revision is <b>not</b>
     * considered a revision to be considered.<br>
     * <b>This method may be called by the <i>IngestMultiplexerServer</i></b>, <i>not</i> by the ASC<br>
     * <br>
     * This method may take some time to finish. It is tried to distinguish between inconsistencies triggered
     * by programming errors and inconsistencies in the file system. In case of a programming error an
     * exception is thrown. In case of a file system inconsistency, it is tried to correct the error.<br>
     * 
     * @return list of all those revisions whose ingest have not terminated yet. <em>This list is <b>sorted</b> w.r.t. providerId, ingestId and revisionId.</em>
     */
    public synchronized List<SipReader> returnNotTerminatedIngests(List<ProductionLine> productiveSystems) {
        if ( EXCESSIVE_DEBUG ) {
            LOG.info("production lines are:");
            for ( ProductionLine productionLine : productiveSystems ) {
                LOG.info("  " + productionLine.getName());
            }
        }
        List<SipReader> revisionsNotFinished = new ArrayList<SipReader>();
        try {
            List<File> providers = getSortedChildren(this.dataPath, "invalid root directory");
            for ( File provider : providers ) {
                String providerId = provider.getName();
                List<File> ingests = getSortedChildren(provider, "invalid directory at provider level");
                for ( File ingest : ingests ) {
                    String ingestId = ingest.getName();
                    List<File> revisions = getSortedChildren(ingest, "invalid directory at ingest level");
                    int lastRevisionIndex = revisions.size() - 1;
                    if ( revisions.size() > 1 ) {
                        if ( EXCESSIVE_DEBUG ) {
                            LOG.info(provider.getName()
                                + " : "
                                + ingest.getName()
                                + " has more than 1 revision - old revisions will be checked for termination");
                        }
                        for ( int i = 0; i < lastRevisionIndex; i++ ) {
                            File revision = revisions.get(i);
                            List<BookkeepingData> list = this.bkManager.getBookkeepingDataByIngest(providerId, ingestId);
                            RevisionState revState = checkRevision(list, revision);
                            if ( EXCESSIVE_DEBUG ) {
                                LOG.info(provider.getName()
                                    + " : "
                                    + ingest.getName()
                                    + " : "
                                    + revision.getName()
                                    + " expected state: "
                                    + RevisionState.ALL_PROD_LINES_FINISHED_WO_ERROR
                                    + ", actual state: "
                                    + revState);
                            }
                            Check.isEqual(revState, RevisionState.ALL_PROD_LINES_FINISHED_WO_ERROR, "inconsistency found for "
                                + provider.getName()
                                + " : "
                                + ingest.getName()
                                + " : "
                                + revision.getName());
                        }
                    }

                    if ( revisions.size() > 0 ) {
                        File lastRevision = revisions.get(lastRevisionIndex);
                        String revisionId = lastRevision.getName();

                        List<BookkeepingData> list = this.bkManager.getBookkeepingDataByIngest(providerId, ingestId);
                        RevisionState revisionState = checkRevision(list, lastRevision);

                        if ( (revisionState == RevisionState.AT_LEAST_ONE_PROD_LINE_NOT_SUCCESSFUL) || (revisionState == RevisionState.NOT_COMPLETE) ) {
                            Map<String, BookkeepingWrapper> wrappers = getBookkeepingForAllProdLines(providerId, ingestId, revisionId, productiveSystems);
                            revisionsNotFinished.add(new SipReader(wrappers, lastRevision, providerId, ingestId, lastRevision.getName()));

                            if ( EXCESSIVE_DEBUG ) {
                                LOG.info("this ingest is not yet finished: " + providerId + " : " + ingestId + " : " + lastRevision.getName());
                            }
                        } else {
                            if ( EXCESSIVE_DEBUG ) {
                                LOG.info("this ingest is complete and thus ignored: " + providerId + " : " + ingestId + " : " + lastRevision.getName());
                            }
                        }
                    }

                }
            }
            for ( SipReader ur : revisionsNotFinished ) {
                LOG.info("Ingest of revision not yet finished: " + ur);
            }
            return revisionsNotFinished;
        } catch ( Exception e ) {
            LOG.error("Sip directory structure is inconsistent", e);
            return Collections.emptyList();
        }
    }

    private Map<String, BookkeepingWrapper> getBookkeepingForAllProdLines(
        String providerId,
        String ingestId,
        String revisionId,
        List<ProductionLine> productiveSystems) {
        Map<String, BookkeepingWrapper> wrappers = new HashMap<>();
        for ( ProductionLine line : productiveSystems ) {
            String name = line.getName();
            BookkeepingWrapper wrapper = this.factory.get(providerId, ingestId, revisionId, name);
            if ( wrapper != null ) {
                wrappers.put(name, wrapper);
            }
        }
        return wrappers;
    }

    /**
     * return ALL ingest data as it is stored in the IM's file system as an JSON array of JSON arrays
     * 
     * @return a String describing the state of ingests specified by providerId, ingestId and revisionId.
     */
    public synchronized String getAllIngestsFromDB() {
        JSONObject resultObject = new JSONObject();
        try {
            JSONArray resultArray = new JSONArray();
            List<BookkeepingData> bookkeepingData = this.bkManager.getAllBookkeepingData();
            for ( BookkeepingData data : bookkeepingData ) {
                JSONArray ingestArray = new JSONArray();
                ingestArray.put(""); // Placeholder for the name of the provider
                ingestArray.put(data.getProviderId());
                ingestArray.put(data.getIngestId());
                ingestArray.put(data.getRevisionId());
                ingestArray.put(data.getPoductionLine());
                addRevisionInfo(ingestArray, data);
                resultArray.put(ingestArray);
            }

            resultObject.put("result", resultArray);

        } catch ( Exception e ) {
            LOG.error("exception", e);
            return "";
        }

        return resultObject.toString();
    }

    public synchronized void deleteFsAndBookeeping(String providerId, String ingestId, String revisionId) {
        Check.notEmpty(providerId);
        Check.notEmpty(ingestId);
        Check.notEmpty(revisionId);

        List<File> providers = getSortedChildren(this.dataPath, "invalid root directory");
        File provider = selectFile(providerId, providers);
        List<File> ingests = getSortedChildren(provider, "provider directory does not exist");
        File ingest = selectFile(ingestId, ingests);
        List<File> revisions = getSortedChildren(ingest, "ingest directory does not exist");
        File revision = selectFile(revisionId, revisions);
        File sip = checkChildDirectory(revision, DIR_SIP, "revision directory does not exist");
        File bookkeeping = checkChildDirectory(revision, DIR_BOOKKEEPING, "bookkeeping directory does not exist");

        List<BookkeepingData> data = this.bkManager.getBookkeepingDataByRevision(providerId, ingestId, revisionId);

        String revisionPath = revision.getAbsolutePath();
        if ( !sip.isDirectory() ) {
            LOG.error("deleteFS has NOT deleted the revision (IM DATA WAS already INCONSISTENT): " + revisionPath);
        } else if ( !bookkeeping.isDirectory() ) {
            LOG.error("deleteFS has NOT deleted the revision (IM DATA WAS already INCONSISTENT): " + revisionPath);
        } else if ( !mayRevisionBeRemoved(data) ) {
            LOG.error("deleteFS has NOT deleted the revision (FINISH or ABORT the revision first (on all production lines)): " + revisionPath);
        } else {
            try {
                this.bkManager.deleteBookkeepingDataByRevision(providerId, ingestId, revisionId);
                String canonicalPrefix = revision.getCanonicalPath(); // for messages

                revision.setWritable(true); // hopefully was the write permission changed, if not: an error will occur later ...

                boolean sipDeleted = Files.deleteFolderRecursive(sip, canonicalPrefix, true);
                boolean bookkeepingDeleted = Files.deleteFolderRecursive(bookkeeping, canonicalPrefix, true);
                if ( sipDeleted && bookkeepingDeleted ) {
                    ingest.setWritable(true); // hopefully was the write permission changed, if not: an error will occur later ...
                    boolean revisionDeleted = revision.delete();
                    if ( revisionDeleted ) {
                        //LOG.info("deleteFS removed the revision: " + revisionPath);
                        if ( ingest.list().length == 0 ) {
                            provider.setWritable(true); // hopefully was the write permission changed, if not: an error will occur later ...
                            boolean ingestDeleted = ingest.delete();
                            if ( ingestDeleted ) {
                                // LOG.info("deleteFS removed the ingest: " + revisionPath);
                                if ( provider.list().length == 0 ) {
                                    this.dataPath.setWritable(true); // hopefully was the write permission changed, if not: an error will occur later ...
                                    boolean providerDeleted = provider.delete();
                                    if ( providerDeleted ) {
                                        // LOG.info("deleteFS removed the provider: " + revisionPath);
                                    } else {
                                        LOG.error("deleteFS could not delete the provider directory (IM DATA is CONSISTENT): " + revisionPath);
                                    }
                                    this.dataPath.setWritable(false); // hopefully was the write permission changed ...
                                } else {
                                    LOG.info("deleteFS keeps the provider, further ingests exist: " + revisionPath);
                                }
                            } else {
                                LOG.error("deleteFS could not delete the ingest directory (IM DATA is CONSISTENT): " + revisionPath);
                            }
                            provider.setWritable(false); // hopefully was the write permission changed ...
                        } else {
                            LOG.info("deleteFS keeps the ingests, further revisions exist: " + revisionPath);
                        }
                    } else {
                        LOG.error("deleteFS could not delete the revision directory (IM DATA IS now INCONSISTENT): " + revisionPath);
                    }
                    ingest.setWritable(false); // hopefully was the write permission changed ...

                } else {
                    LOG.error("deleteFS could not delete the sip directory (IM DATA is NOW INCONSISTENT): " + revisionPath);
                }
            } catch ( IOException e ) {
                LOG.error("deleteFS has NOT deleted the revision (canonical path error): " + revisionPath);
            }
        }

    }

    private boolean mayRevisionBeRemoved(List<BookkeepingData> bookkeepingData) {
        for ( BookkeepingData data : bookkeepingData ) {
            if ( !mayRevisionBeRemoved(data) ) {
                return false;
            }
        }
        return true;
    }

    /**
     * <p>
     * Compares the DDB identifiers of the SIPs present in two revision directories with each other. The comparison results are stored to disk and optionally
     * returned.
     * </p>
     * <p>
     * The revision directories are both identified by provider ID, ingest ID and revision ID. The first directory is taken as reference and the results are the
     * changes in the second directory relative to the first.
     * </p>
     * 
     * @param providerId1
     * @param ingestId1
     * @param revisionId1
     * @param providerId2
     * @param ingestId2
     * @param revisionId2
     * @param returnResults If {@code true}, the comparison result will be returned.
     * @return If {@code returnResults}, a JSON string containing an object with the fields "added", "deleted" and "same", containing arrays with the DDB
     *         identifiers added, deleted and
     *         the same. If an exception occurs, an empty string is returned.
     */
    public String compareRevisionDirectories(
        final String providerId1,
        final String ingestId1,
        final String revisionId1,
        final String providerId2,
        final String ingestId2,
        final String revisionId2,
        final boolean returnResults) {

        try {
            List<File> providers = getSortedChildren(this.dataPath, "invalid root directory");

            Check.notEmpty(providerId1);
            Check.notEmpty(ingestId1);
            Check.notEmpty(revisionId1);
            File provider1 = selectFile(providerId1, providers);
            List<File> ingests1 = getSortedChildren(provider1, "provider directory 1 does not exist");
            File ingest1 = selectFile(ingestId1, ingests1);
            List<File> revisions1 = getSortedChildren(ingest1, "ingest directory 1 does not exist");
            File revision1 = selectFile(revisionId1, revisions1);
            File dir1 = checkChildDirectory(revision1, DIR_SIP, "revision directory 1 does not exist");

            Check.notEmpty(providerId2);
            Check.notEmpty(ingestId2);
            Check.notEmpty(revisionId2);
            File provider2 = selectFile(providerId2, providers);
            List<File> ingests2 = getSortedChildren(provider2, "provider directory 2 does not exist");
            File ingest2 = selectFile(ingestId2, ingests2);
            List<File> revisions2 = getSortedChildren(ingest2, "ingest directory 2 does not exist");
            File revision2 = selectFile(revisionId2, revisions2);
            File dir2 = checkChildDirectory(revision2, DIR_SIP, "revision directory 2 does not exist");

            // Get the IDs.
            List<String> ids1 = getSipsInDirectory(dir1);
            List<String> ids2 = getSipsInDirectory(dir2);

            // Compare the IDs.
            ComparisonResult<String> comparisonResult = new ListComparator<String>().compare(ids1, ids2);

            // Create a JSON result.
            JSONArray added = new JSONArray(comparisonResult.getAdded());
            JSONArray deleted = new JSONArray(comparisonResult.getDeleted());
            JSONArray same = new JSONArray(comparisonResult.getSame());
            JSONObject resultObject = new JSONObject();
            resultObject.put("added", added);
            resultObject.put("deleted", deleted);
            resultObject.put("same", same);
            String jsonResult = resultObject.toString();

            // Write the comparison to disk.
            String comparisonFileName =
                "p"
                    + providerId1
                    + "_r"
                    + ingestId1
                    + "_i"
                    + revisionId1
                    + "-p"
                    + providerId2
                    + "_r"
                    + ingestId2
                    + "_i"
                    + revisionId2
                    + "-"
                    + COMPARISON_DATE_FORMAT.get().format(new Date())
                    + ".json";
            File comparisonFile = new File(this.comparisonPath.getAbsolutePath() + File.separator + comparisonFileName);
            OutputStreamWriter out = null;
            try {
                out = new OutputStreamWriter(new FileOutputStream(comparisonFile), "UTF-8");
                out.write(jsonResult);
            } finally {
                IOUtils.closeQuietly(out);
            }

            // If requested, return the comparison.
            if ( returnResults ) {
                return jsonResult;
            } else {
                return "";
            }
        } catch ( Exception e ) {
            LOG.error(e.getLocalizedMessage(), e);
            return "";
        }
    }

    /**
     * @return The data path
     */
    public File getDataPath() {
        return this.dataPath;
    }

    /**
     * returns the file from a list of files that has the name supplied as parameter
     * 
     * @param name the name of the file looked for
     * @param files the files to be checked
     * @return the file from files that has a matching name
     */
    private static File selectFile(String name, List<File> files) {
        for ( File file : files ) {
            if ( file.getName().equals(name) ) {
                return file;
            }
        }
        return null;
    }

    /**
     * check whether all revisions of an ingest have finished
     * 
     * @param wrappers
     * @param revisionId
     * @param ingestId
     * @param providerId
     * @param ingest the directory containing all persistent data w.r.t. an ingest
     * @param productionLines
     * @param revisionId
     * @param providerId
     * @param productionLines the production lines to be checked. Not null.
     * @return null if all revisions for the ingest specified have terminated, return a error message
     *         otherwise.
     */
    private String haveAllRevisionsTerminated(File ingest, String providerId, String revisionId, List<ProductionLine> productionLines) {
        Check.isTrue(ingest.isDirectory());
        Check.notNull(productionLines);
        File[] children = ingest.listFiles();
        Arrays.sort(children);

        for ( File revision : children ) {
            if ( revision.getName().equals(revisionId) ) {
                return "Revision already exists";
            }
        }

        List<BookkeepingData> list = this.bkManager.getBookkeepingDataByIngest(providerId, ingest.getName());

        if ( !list.isEmpty() ) {
            for ( File revision : children ) {
                //check existing bookkeeping for the given revision
                if ( checkRevision(list, revision) != RevisionState.ALL_PROD_LINES_FINISHED_WO_ERROR ) {
                    return "A revision not yet finished was found, a new revision cannot be created";
                }
            }
        }
        return null;
    }

    /**
     * when the file system ist checked w.r.t. its consistency or the state of the various ingests, this enum
     * defines the possible outcomes of this check.
     */
    private static enum RevisionState {
        INCONSISTENT, ASC_DID_NOT_TERMINATE, NOT_COMPLETE, ALL_PROD_LINES_FINISHED_WO_ERROR, AT_LEAST_ONE_PROD_LINE_NOT_SUCCESSFUL
    }

    /**
     * check a revision and return whether the revision has been ingested completely on all production lines<br>
     * 
     * @param bkManager
     * @param revisionId
     * @param ingestId
     * @param providerId
     * @param revision file to be checked
     * @param productiveSystems
     * @param productionLines the names of the productive systems. If null, the consistency of the bookkeeping
     *        files w.r.t. production lines is <b>not</b> checked.
     * @return null if all bookkeeping data is correct and both ASC and all production lines have terminated,
     *         otherwise return an error message.
     */
    private RevisionState checkRevision(List<BookkeepingData> list, File revision) {
        RevisionState result = RevisionState.ALL_PROD_LINES_FINISHED_WO_ERROR;

        final String revisionPath = revision.getAbsolutePath();
        if ( !revision.isDirectory() ) {
            LOG.error("SEVERE ERROR: Revision " + revisionPath + " inconsistent (1)");
            return RevisionState.INCONSISTENT;
        }

        if ( (list == null) || list.isEmpty() ) {
            LOG.error("SEVERE ERROR: Revision inconsistent (2): bookkeeping data is missing");
            return RevisionState.INCONSISTENT;
        }

        for ( BookkeepingData data : list ) {
            String pl = data.getPoductionLine();

            AscState ascState = AscState.valueOf(data.getAscState());
            if ( ascState == null ) {
                LOG.error("SEVERE ERROR: Revision " + revisionPath + " inconsistent : ASC State is missing");
                return RevisionState.INCONSISTENT;
            }

            if ( data.getAscTotal() < 0 ) {
                LOG.error("SEVERE ERROR: Revision " + revisionPath + " inconsistent : ASC total is missing");
                return RevisionState.INCONSISTENT;
            }

            if ( ascState == AscState.REVISION_IN_CONSTRUCTION ) {
                LOG.info("Revision " + revisionPath + " in construction - is ignored");
                return RevisionState.ASC_DID_NOT_TERMINATE;
            }
            if ( data.getCurrent() < 0 ) {
                logError("current_" + pl + " mismatch in " + revisionPath);
                data.setCurrent(0L);
            }
            if ( data.getIngested() == null ) {
                logError("ingested_" + pl + " mismatch in " + revisionPath);
                data.setIngested("");
            }

            IngestState ingestState = IngestState.valueOf(data.getIngestState());
            if ( ingestState == null ) {
                logError("state_" + pl + " mismatch in " + revisionPath);
                data.setIngestState(IngestState.READY_TO_START.toString());
            }

            if ( ingestState == IngestState.FINISHED_ERROR ) {
                result = RevisionState.AT_LEAST_ONE_PROD_LINE_NOT_SUCCESSFUL;
            } else if ( (ingestState == IngestState.FINISHED) || (ingestState == IngestState.ABORTED) ) {
                // NOSONAR: treat that as OK (for further ingests)
            } else {
                return RevisionState.NOT_COMPLETE;
            }

            this.bkManager.updateBookkeepingData(data);
        }

        return result;
    }

    /**
     * log an error
     * 
     * @param errorMsgPart suffix of the error message
     */
    private static void logError(String errorMsgPart) {
        String errorMsg = "Sip directory structure inconsistent. " + errorMsgPart;
        LOG.error(errorMsg);
    }

    /**
     * check a revision and return true if bookkeeping data is correct and IngestServer doesn't work on data of this revision.
     */
    private boolean mayRevisionBeRemoved(BookkeepingData data) {
        switch ( IngestState.valueOf(data.getIngestState()) ) {
            case FINISHED:
            case FINISHED_ERROR:
            case ABORTED:
                break;
            default:
                return false;

        }
        return true;
    }

    /**
     * add information about a revision to a JSON array
     * 
     * @param ingestJson the array, to which properties have to be appended
     * @param data the revision, from which information has to be extracted
     * @param productionLines all production lines to be considered
     * @throws JSONException should not occur :-)
     */
    private void addRevisionInfo(JSONArray ingestJson, BookkeepingData data) throws JSONException {
        String providerString = data.getZProvider();
        JSONObject providerJson = new JSONObject(providerString);
        ingestJson.put(0, providerJson.getString("source_folder") + " <small>(" + providerJson.getString("provider_name") + ")</small>");
        ingestJson.put(data.getAscState());
        ingestJson.put(data.getAscTotal());
        Calendar cal = new GregorianCalendar(Locale.GERMANY);
        cal.setTimeInMillis(data.getCreated());
        DateFormat formatter = new SimpleDateFormat("yyyy.MM.dd HH:mm:ss");
        ingestJson.put(formatter.format(cal.getTime()));
        ingestJson.put(mayRevisionBeRemoved(data) ? "may be deleted" : "IMPOSSIBLE TO DELETE");
    }

    /**
     * extract information about a revision and store that information into a JSON object-
     * 
     * @param revisionJson the response object, into which properties have to be inserted
     * @param revision the revision, from which information has to be extracted
     * @throws JSONException should not occur :-)
     */
    private void getRevisionInfo(JSONObject revisionJson, List<BookkeepingData> revisions) throws JSONException {
        if ( revisions == null ) {
            return;
        }

        for ( BookkeepingData revision : revisions ) {
            revisionJson.put("generated", System.currentTimeMillis());
            revisionJson.put("asc", revision.getIngestState());
            revisionJson.put("total", revision.getAscTotal());

            String providerString = revision.getZProvider();
            JSONObject providerJson = new JSONObject(providerString);
            revisionJson.put("provider", providerJson);

            revisionJson.put("created", revision.getCreated());
            revisionJson.put("state", revision.getIngestState());
            revisionJson.put("current", revision.getCurrent());
            revisionJson.put("ingested", revision.getIngested());
        }
    }

    /**
     * make sure that a child directory as part of a parent directory does exist.<br>
     * - if the directory does exist, everything is fine.<br>
     * - if it is not found, create the child directory. The parent directory gets READ permission. The child
     * directory get the permission as requested by parameter {@code childREAD}.
     * 
     * @param parent the parent directory
     * @param childName the name of the child directory
     * @param childRead true, if read permission for the child directory should be set; false, if write
     *        permission should be set
     * @return
     */
    private static File makeChildDirectoryAvailable(File parent, String childName, boolean childRead) {
        Check.isTrue(parent.isDirectory(), "Must be a directory: " + parent.getAbsolutePath());
        File child = new File(parent, childName);

        if ( child.exists() ) {
            Check.isTrue(child.isDirectory());
        } else {
            try {
                setWritable(parent, true);
                Check.isTrue(child.mkdir());
                Check.isTrue(setWritable(child, !childRead));
            } finally {
                setWritable(parent, false);
            }
        }
        return child;
    }

    /**
     * make sure that a child directory does exist.
     * 
     * @param parent the parent directory
     * @param childName the name of the child directory
     * @param errorMsg the error message to use if the child directory does not exist
     * @return the existing directory
     */
    private static File checkChildDirectory(File parent, String childName, String errorMsg) {
        Check.notNull(parent, errorMsg);
        Check.isTrue(parent.exists(), errorMsg);
        Check.isTrue(parent.isDirectory(), errorMsg);
        File child = new File(parent, childName);
        Check.isTrue(child.isDirectory(), errorMsg);
        return child;
    }

    /**
     * Sets the "writable" attribute for everyone on a file or directory.
     * <p>
     * On Windows machines do nothing to avoid a problem with:<br>
     * http://stackoverflow.com/questions/5302269/java-file-setwritable-and-stopped-working-correctly-after-jdk-6u18
     * </p>
     * 
     * @param f the {@link File} to change the "writable" attribute of
     * @param writable the new value of the "writable" attribute
     * @return true if this call succeeded, false if not
     */
    private static boolean setWritable(File f, boolean writable) {
        return IS_WINDOWS || f.setWritable(writable, false);
    }

    /**
     * return the files from a directory sorted by their respecting names.<br>
     * Visibility could be reduced ...
     * 
     * @param base the directory whose children are requested
     * @return files sorted ascending by their names
     */
    public static List<File> getSortedChildren(File base, String errorMsg) {
        Check.isTrue((base != null) && base.isDirectory(), errorMsg);
        File[] children = base.listFiles();
        Arrays.sort(children);

        if ( EXCESSIVE_DEBUG ) {
            LOG.info("getSortedChildren for path " + base + " got:");
            for ( File child : children ) {
                LOG.info("  " + child.getName());
            }
        }

        return Arrays.asList(children);
    }

    /**
     * guarantee, that a string only contains the word characters [a-zA-Z_0-9]
     * 
     * @param toBeChecked the string to be checked
     */
    private void checkSipNames(String toBeChecked) {
        Check.notNull(toBeChecked);
        Matcher m = PATH_PART.matcher(toBeChecked);
        Check.isTrue(m.matches());
    }

    /**
     * Returns a list of SIPs in a directory, i.e. all files with a name ending with ".sip". The returned list contains the file names without ".sip"
     * 
     * @param directory The directory to list. The directory must exist and be readable.
     * @throws IOException If the directory or a file cannot be read.
     */
    List<String> getSipsInDirectory(File directory) throws IOException {
        // Collect the SIP IDs efficiently in a LinkedList.
        final List<String> idsForFilling = new LinkedList<String>();
        SimpleFileVisitor<Path> getAllSips = new SimpleFileVisitor<Path>() {
            @Override
            public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException {
                String fileName = file.getName(file.getNameCount() - 1).toString();
                if ( fileName.endsWith(".sip") ) {
                    idsForFilling.add(fileName.substring(0, fileName.length() - 4));
                }
                return FileVisitResult.CONTINUE;
            }
        };
        java.nio.file.Files.walkFileTree(directory.toPath(), getAllSips);

        // Now, for efficient access, copy the SIP IDs into an ArrayList.
        List<String> ids = new ArrayList<String>(idsForFilling);
        return ids;
    }

    /**
     * Returns provider information stored in bookkeeping data
     * 
     * @param providerId
     * @param ingestId
     * @param revisionId
     * @param productionLine
     * @return
     */
    public String getProviderDescription(String providerId, String ingestId, String revisionId, String productionLine) {
        BookkeepingData data = null;
        if ( !Strings.isNullOrEmpty(productionLine) ) {
            data = this.bkManager.getBookkeepingDataByPl(providerId, ingestId, revisionId, productionLine);
        } else {
            List<BookkeepingData> list = this.bkManager.getBookkeepingDataByRevision(providerId, ingestId, revisionId);
            if ( !list.isEmpty() ) {
                data = list.get(0);
            }
        }
        if ( data != null ) {
            return data.getZProvider();
        }
        return null;
    }
}