text:
/*
 * CALCompiler.java
 * Created: Feb 23, 2000
 * By: Luke Evans
 */

package org.openquark.cal.compiler;

import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStream;
import java.io.Reader;
import java.io.StringReader;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.SortedMap;
import java.util.TreeMap;
import java.util.logging.Level;
import java.util.logging.Logger;

import org.openquark.cal.compiler.CompilerMessage.AbortCompilation;
import org.openquark.cal.internal.serialization.ModuleSerializationTags;
import org.openquark.cal.internal.serialization.RecordInputStream;
import org.openquark.cal.machine.GeneratedCodeInfo;
import org.openquark.cal.machine.Module;
import org.openquark.cal.module.Cal.Core.CAL_Prelude;
import org.openquark.cal.services.Status;
import org.openquark.cal.util.Graph;
import org.openquark.cal.util.Vertex;
import org.openquark.cal.util.VertexBuilder;
import org.openquark.cal.util.VertexBuilderList;
import org.openquark.util.Pair;

/**
 * This is an instance of the CAL compiler.
 * Only one compilation job can be active per CALCompiler object.
 * Creation date: (2/23/00 5:53:47 AM)
 * @author LWE
 */
final class CALCompiler {

    /** The namespace for log messages from the compiler package. */
    static final private String COMPILER_LOGGER_NAMESPACE = "org.openquark.cal.compiler";

    /** An instance of the Java logger used to log compiler messages. */
    static final Logger COMPILER_LOGGER = Logger.getLogger(COMPILER_LOGGER_NAMESPACE);

    static {
        COMPILER_LOGGER.setLevel(Level.FINEST);
    }

    // Compilation state

    /**
     * The multiplexed lexer for the CAL grammar.
     *
     * A multiplexed lexer encapsulates the multiplexing of the two lexers used
     * in lexing CAL source: the main CAL lexer and the CALDoc lexer. Since
     * these two lexers are codependent (namely on scanning '/''*''*' as the
     * start of a CALDoc comment, and '*''/' as the end of one), the two lexers
     * should not be created or accessed individually, but rather through this
     * field, which guarantees that they are always created and used as a pair.
     */
    private CALMultiplexedLexer lexer;

    private final CALParser parser;

    /** Recognizes valid ASTs. */
    private final CALTreeParser treeParser;

    private final CALTypeChecker typeChecker;

    /**
     * An instance of the deprecation scanner which performs a pass for finding deprecated entities.
     */
    private final DeprecationScanner deprecationScanner;

    /** The Packager */
    private Packager packager;

    /** Object to log messages generated by compiler or subordinates.  Never null. */
    private CompilerMessageLogger msgLogger = new MessageLogger();

    /** The time stamp for the source of the currently compiling module. */
    private long currentModuleTimeStamp;

    /**
     * used to debug the SourceModel model of CAL's source - how this is done depends on
     * the value of DEBUG_SOURCE_MODEL_TEXT
     */
    static final private boolean DEBUG_SOURCE_MODEL = false;

    /**
     * If true, the source model is debugged by converting every successfully parsed module
     * into a SourceModel, then converting the SourceModel to module text,
     * and trying to parse this again.
     * If false, it is debugged by using toParseTree to go directly back to a ParseTreeNode.
     *
     * Only used if DEBUG_SOURCE_MODEL is true.
     */
    static final private boolean DEBUG_SOURCE_MODEL_TEXT = false;

    /**
     * If true, a SourceModelCopier is used to make a deep copy of the source model,
     * and other visitors are created and tested on the source model.
     *
     * Only used if DEBUG_SOURCE_MODEL is true.
     */
    static final private boolean DEBUG_SOURCE_MODEL_VISITOR = false;

    /**
     * If true, the all source model debugging code paths are enabled.
     */
    private static boolean inUnitTestDebugSourceModelMode = false;

    /**
     * If true, instrumentation of concurrent compilation is enabled.
     */
    private static final boolean DEBUG_CONCURRENT_COMPILATION = false;

    /**
     * Private mutex used for locking instrumentation information shared across all instances of this class.
     * Using a zero-length byte[] instead of a new Object() is a small optimization.
     */
    private static final byte[] classMutex = new byte[0];

    /**
     * The number of active threads currently compiling across all instances of this class.
     */
    private static int numThreadsCompiling = 0;

    /**
     * The total number of adjuncts compiled across all instances of this class.
     */
    private static int adjunctCompileCount = 0;

    /**
     * The total number of modules compiled across all instances of this class.
     */
    private static int moduleCompileCount = 0;

    /**
     * A simple container class to hold info about a single import into a module.
     * @author Edward Lam
     */
    private static class ImportNodeInfo {
        private final ModuleName importedName;
        private final ParseTreeNode importedModuleNameNode;

        /**
         * Constructor for a ImportNodeInfo.
         * Use this if there no corresponding parse tree node (e.g. defined via source model.).
         * @param importedName the name of the imported module.
         */
        private ImportNodeInfo(SourceModel.Name.Module importedName) {

            if (importedName == null) {
                throw new NullPointerException();
            }
            this.importedName = ModuleName.make(importedName.toSourceText());

            this.importedModuleNameNode = null;
        }

        /**
         * Constructor for a ImportNodeInfo.
         * Use this if there no corresponding parse tree node (e.g. defined via source model.).
         * @param importedName the name of the imported module.
         */
        private ImportNodeInfo(ModuleName importedName) {

            if (importedName == null) {
                throw new NullPointerException();
            }
            this.importedName = importedName;

            this.importedModuleNameNode = null;
        }

        /**
         * Constructor for a ImportNodeInfo from a module import declaration.
         * @param importDeclarationNode the parse tree node corresponding to the module import declaration.
         */
        private ImportNodeInfo(ParseTreeNode importDeclarationNode) {

            if (importDeclarationNode == null) {
                throw new NullPointerException();
            }
            importDeclarationNode.verifyType(CALTreeParserTokenTypes.LITERAL_import);

            this.importedModuleNameNode = importDeclarationNode.firstChild();

            this.importedName = ModuleNameUtilities.getModuleNameFromParseTree(importedModuleNameNode);

            if (this.importedName == null) {
                throw new NullPointerException();
            }
        }

        /**
         * @return the name of the imported module.
         */
        public ModuleName getImportName() {
            return importedName;
        }

        /**
         * @return the parse tree node corresponding to the module name in the imported module declaration,
         *   or null if there was no corresponding parse tree.
         */
        public ParseTreeNode getImportedModuleNameNode() {
            return importedModuleNameNode;
        }
    }

    /**
     * A class to hold a CompiledModuleSourceDefinition, plus info to determine its validity
     * gleaned by partial deserialization of its input stream.
     *
     * Previously, this info was obtained by just partially deserializing the input stream to find the relevant info
     *  (for instance, the timestamp) wherever it was needed.  This caused the input stream to be opened multiple times,
     *  which is quite slow if the input stream is coming from disk.
     *
     * @author Edward Lam
     */
    private static class CompiledDefinitionInfo {
        /** The CompiledModuleSourceDefinition for this info. */
        private final CompiledModuleSourceDefinition cmsd;

        /** The timestamp from the compiled module source, or -1 if this has not yet been determined. */
        private long timeStamp = -1;

        /** (Set of ModuleName) The imported modules names from the compiled module source, or null if this has not yet been determined. */
        private Set<ModuleName> importedModuleNames;

        /** The generated code info from the compiled module source, or null if this has not yet been determined. */
        private GeneratedCodeInfo codeInfo;

        /**
         * Constructor for a CompiledDefinitionInfo.
         * @param cmsd
         */
        CompiledDefinitionInfo(CompiledModuleSourceDefinition cmsd) {
            this.cmsd = cmsd;
        }

        /**
         * Read the header info from the CompiledModuleSourceDefinition if it hasn't already been read.
         * @throws IOException
         */
        private void initCompiledDefinitionHeaderInfo() throws IOException {

            // Check whether initialization has already occurred.
            if (importedModuleNames != null) {
                return;
            }

            Status status = new Status("get imports status");
            InputStream is = cmsd.getInputStream(status);

            if (is == null) {
                throw new IOException ("Error reading compiled definition for module " + cmsd.getModuleName() + " : " + status.getMessage());
            }

            RecordInputStream ris = new RecordInputStream(is);

            try {
                // read the timestamp.
                ris.findRecord(ModuleSerializationTags.SERIALIZATION_INFO);
                this.timeStamp = ris.readLong();

                // read the code info
                this.codeInfo = Module.loadGeneratedCodeInfo(ris);
                ris.skipRestOfRecord();

                // Read the imports information.
                this.importedModuleNames = Module.readDependencies(ris);

            } finally {
                try {
                    ris.close();
                } catch (IOException e) {
                    // Ignore this particular exception.
                }
            }
        }

        /**
         * @return the CompiledModuleSourceDefinition for this info.
         */
        public CompiledModuleSourceDefinition getCompiledModuleSourceDefinition() {
            return cmsd;
        }

        /**
         * @return The timestamp from the compiled module source.
         * @throws IOException if there was a problem reading the compiled module source.
         */
        long getTimeStamp() throws IOException {
            initCompiledDefinitionHeaderInfo();
            return timeStamp;
        }

        /**
         * @return (Set of ModuleName) The imported modules names from the compiled module source.
         * @throws IOException if there was a problem reading the compiled module source.
         */
        Set<ModuleName> getImportedModuleNames() throws IOException {
            initCompiledDefinitionHeaderInfo();
            return new HashSet<ModuleName>(importedModuleNames);
        }

        /**
         * @return The generated code info from the compiled module source.
         * @throws IOException if there was a problem reading the compiled module source.
         */
        GeneratedCodeInfo getCodeInfo() throws IOException {
            initCompiledDefinitionHeaderInfo();
            return codeInfo;
        }
    }

    /**
     * A class to hold info about a number of module source definitions, plus info about whether those definitions are empty.
     *
     * Previously, this info was obtained wherever it was needed, potentially causing multiple disk accesses if that definition were on disk.
     *
     * @author Edward Lam
     */
    private static class SourceDefinitionsInfo {
        /** (ModuleName->ModuleSourceDefinition) Map from module name to source definition. */
        private final Map<ModuleName, ModuleSourceDefinition> sourceNameToDefinitionMap;

        /** (ModuleSourceDefinition->Boolean) Map from source defn to whether it is empty. */
        private final Map<ModuleSourceDefinition, Boolean> sourceDefinitionToEmptyMap = new HashMap<ModuleSourceDefinition, Boolean>();

        /**
         * Constructor for a SourceDefinitionsInfo.
         * @param sourceNameToDefinitionMap
         */
        SourceDefinitionsInfo(Map<ModuleName, ModuleSourceDefinition> sourceNameToDefinitionMap) {
            this.sourceNameToDefinitionMap = new HashMap<ModuleName, ModuleSourceDefinition>(sourceNameToDefinitionMap);
        }

        /**
         * @param moduleName the name of a module
         * @return the corresponding ModuleSourceDefinition, or null if the module is not the name of a module in this info.
         */
        public ModuleSourceDefinition getDefinition(ModuleName moduleName) {
            return sourceNameToDefinitionMap.get(moduleName);
        }

        /**
         * @return the ModuleSourceDefinitions in this info object.
         */
        public Collection<ModuleSourceDefinition> getSourceDefinitions() {
            return sourceNameToDefinitionMap.values();
        }

        /**
         * Determines whether the module source is in fact empty.
         * If this method is called multiple times, the module source definition will be opened only the first time to determine whether it is empty.
         *
         * @param msd the module source.
         * @return true if the module source is empty.
         */
        public boolean isEmptyModuleSource(ModuleSourceDefinition msd) {
            /*
             * TODOEL: Would it make a significant difference if we instead had this method in ModuleSourceDefinition?
             * ie. if there were an abstract method ModuleSourceDefinition.isEmptyModuleSource().
             */

            // Check for a value in the map.
            Boolean bool = sourceDefinitionToEmptyMap.get(msd);

            if (bool == null) {
                InputStream is = msd.getInputStream(new Status("Getting module source input stream"));
                boolean isEmpty = false;
                try {
                    try {
                        if (is.read() == -1) {
                            isEmpty = true;
                        }
                    } finally {
                        is.close();
                    }
                } catch (IOException e) {

                }

                // Add the value to the map.
                bool = Boolean.valueOf(isEmpty);
                sourceDefinitionToEmptyMap.put(msd, bool);
            }

            return bool.booleanValue();
        }
    }

    /**
     * This is a simple container class which hold info about the result of parsing out
     * module headers (ie. module declaration and imports) from a group of module source definitions.
     *
     * @author Edward Lam
     */
    private static class SourceDefinitionsHeaderParseInfo {
        /** The names of modules for which parsing failed. */
        private final Set<ModuleName> nonParseableModuleNamesSet;

        /** (ModuleName->ImportNodeInfo[]) Map from module name to its import info.
          * In order to obtain deterministic ordering of modules in the dependency graph, use a sorted map. */
        private final SortedMap<ModuleName, ImportNodeInfo[]> moduleNameToImportNodeInfoMap;

        /** (ModuleName->ParseTreeNode) Map from module name to its parse tree node, if any.
         * For source positions while error checking. */
        private final Map<ModuleName, ParseTreeNode> moduleNameToHeaderDefnNodeMap;

        SourceDefinitionsHeaderParseInfo(Set<ModuleName> nonParseableModuleNamesSet, SortedMap<ModuleName, ImportNodeInfo[]> moduleNameToImportNodeInfoMap, Map<ModuleName, ParseTreeNode> moduleNameToHeaderDefnNodeMap) {
            this.nonParseableModuleNamesSet = nonParseableModuleNamesSet;
            this.moduleNameToHeaderDefnNodeMap = moduleNameToHeaderDefnNodeMap;
            this.moduleNameToImportNodeInfoMap = moduleNameToImportNodeInfoMap;
        }

        /**
         * @return the moduleNameToHeaderDefnNodeMap
         */
        public Map<ModuleName, ParseTreeNode> getModuleNameToHeaderDefnNodeMap() {
            return moduleNameToHeaderDefnNodeMap;
        }

        /**
         * @return the moduleNameToImportNodeInfoMap
         */
        public SortedMap<ModuleName, ImportNodeInfo[]> getModuleNameToImportNodeInfoMap() {
            return moduleNameToImportNodeInfoMap;
        }

        /**
         * @return the nonParseableModuleNamesSet
         */
        public Set<ModuleName> getNonParseableModuleNamesSet() {
            return nonParseableModuleNamesSet;
        }
    }

    /**
     * This is the abstract base class for the different kinds of statuses that can be returned
     * by the {@link CALCompiler#loadCompiledModule} method.
     *
     * @author Joseph Wong
     */
    private static abstract class CompiledModuleLoadStatus {

        /**
         * Represents the status where an attempt has been made to load the compiled module after it has been located and
         * after it has passed the initial validation checks.
         *
         * @author Joseph Wong
         */
        private static final class LoadAttempted extends CompiledModuleLoadStatus {
            /** Constructs an instance of this class. */
            private LoadAttempted() {}
        }

        /**
         * Represents the status where the compiled module cannot be found.
         *
         * @author Joseph Wong
         */
        private static final class NotFound extends CompiledModuleLoadStatus {
            /** Constructs an instance of this class. */
            private NotFound() {}
        }

        /**
         * Represents the status where the compiled module has a timestamp that is older than that of the corresponding module source.
         *
         * @author Joseph Wong
         */
        private static final class OlderThanSource extends CompiledModuleLoadStatus {
            /** Constructs an instance of this class. */
            private OlderThanSource() {}
        }

        /**
         * Represents the status where one of the modules imported by the compiled module cannot be found.
         *
         * @author Joseph Wong
         */
        private static final class DependeeNotFound extends CompiledModuleLoadStatus {

            /**
             * The name of the dependee module.
             */
            private final ModuleName dependeeName;

            /**
             * Constructs an instance of this class.
             * @param dependeeName the name of the dependee module.
             */
            private DependeeNotFound(final ModuleName dependeeName) {
                if (dependeeName == null) {
                    throw new NullPointerException();
                }

                this.dependeeName = dependeeName;
            }

            /**
             * @return the name of the dependee module.
             */
            private ModuleName getDependeeName() {
                return dependeeName;
            }
        }

        /**
         * Represents the status where the compiled module has a timestamp that is older than that of one of the imported modules.
         *
         * @author Joseph Wong
         */
        private static final class OlderThanDependee extends CompiledModuleLoadStatus {

            /**
             * The name of the dependee module.
             */
            private final ModuleName dependeeName;

            /**
             * Constructs an instance of this class.
             * @param dependeeName the name of the dependee module.
             */
            private OlderThanDependee(final ModuleName dependeeName) {
                if (dependeeName == null) {
                    throw new NullPointerException();
                }

                this.dependeeName = dependeeName;
            }

            /**
             * @return the name of the dependee module.
             */
            private ModuleName getDependeeName() {
                return dependeeName;
            }
        }

        /**
         * Represents the status where an exception is caught or where an internal error has occurred during the processing.
         *
         * @author Joseph Wong
         */
        private static final class ExceptionCaughtOrInternalError extends CompiledModuleLoadStatus {
            /** Constructs an instance of this class. */
            private ExceptionCaughtOrInternalError() {}
        }

        /** Private constructor. For use by subclasses only. */
        private CompiledModuleLoadStatus() {}
    }

    /**
     * This is the abstract base class for the different kinds of statuses that can be returned
     * by timestamp checking methods like {@link CALCompiler#isCompiledSourceUpToDate}.
     *
     * @author Joseph Wong
     */
    private static abstract class TimestampCheckStatus {

        /**
         * Represents the status where the module of interest is up-to-date.
         *
         * @author Joseph Wong
         */
        private static final class UpToDate extends TimestampCheckStatus {
            /** Constructs an instance of this class. */
            private UpToDate() {}
        }

        /**
         * Represents the status where the compiled module has a timestamp that is older than that of the corresponding module source.
         *
         * @author Joseph Wong
         */
        private static final class OlderThanSource extends TimestampCheckStatus {
            /** Constructs an instance of this class. */
            private OlderThanSource() {}
        }

        /**
         * Represents the status where one of the modules imported by the compiled module cannot be found.
         *
         * @author Joseph Wong
         */
        private static final class DependeeNotFound extends TimestampCheckStatus {

            /**
             * The name of the dependee module.
             */
            private final ModuleName dependeeName;

            /**
             * Constructs an instance of this class.
             * @param dependeeName the name of the dependee module.
             */
            private DependeeNotFound(final ModuleName dependeeName) {
                if (dependeeName == null) {
                    throw new NullPointerException();
                }

                this.dependeeName = dependeeName;
            }

            /**
             * @return the name of the dependee module.
             */
            private ModuleName getDependeeName() {
                return dependeeName;
            }
        }

        /**
         * Represents the status where the compiled module has a timestamp that is older than that of one of the imported modules.
         *
         * @author Joseph Wong
         */
        private static final class OlderThanDependee extends TimestampCheckStatus {

            /**
             * The name of the dependee module.
             */
            private final ModuleName dependeeName;

            /**
             * Constructs an instance of this class.
             * @param dependeeName the name of the dependee module.
             */
            private OlderThanDependee(final ModuleName dependeeName) {
                if (dependeeName == null) {
                    throw new NullPointerException();
                }

                this.dependeeName = dependeeName;
            }

            /**
             * @return the name of the dependee module.
             */
            private ModuleName getDependeeName() {
                return dependeeName;
            }
        }

        /**
         * Represents the status where an exception is caught during the processing.
         *
         * @author Joseph Wong
         */
        private static final class ExceptionCaught extends TimestampCheckStatus {
            /** Constructs an instance of this class. */
            private ExceptionCaught() {}
        }

        /** Private constructor. For use by subclasses only. */
        private TimestampCheckStatus() {}
    }

    /**
     * Construct CALCompiler from a Reader.
     */
    CALCompiler() {
        // Do the standard initialisations

        Reader inStream = new StringReader ("");

        // Make a multiplexed lexer
        lexer = new CALMultiplexedLexer(this, inStream, null);

        // Create a recogniser (parser), it gets its tokens from the multiplexed lexer
        parser = new CALParser(this, lexer);
        treeParser = new CALTreeParser(this);

        // The parser creates AST nodes of type ParseTreeNode.
        String treeNodeClassName = ParseTreeNode.class.getName();
        parser.setASTNodeClass(treeNodeClassName);

        typeChecker = new CALTypeChecker(this);

        deprecationScanner = new DeprecationScanner(this);
    }

    static void setInUnitTestDebugSourceModelMode(boolean debugSourceModel) {
        inUnitTestDebugSourceModelMode = debugSourceModel;
    }

    /**
     * Get import info for a module definition defined in a source model.
     * @param moduleDefn the module definition.
     * @return the import info for the module definition.
     */
    private static ImportNodeInfo[] getImportNodeInfo(SourceModel.ModuleDefn moduleDefn) {
        ImportNodeInfo[] importedInfoArray = new ImportNodeInfo[moduleDefn.getNImportedModules()];
        for (int i = 0; i < importedInfoArray.length; i++) {
            importedInfoArray[i] = new ImportNodeInfo(moduleDefn.getNthImportedModule(i).getImportedModuleName());
        }

        return importedInfoArray;
    }

    /**
     * Get import info for a module definition defined in a parse tree.
     * @param moduleDefnNode the module definition.
     * @return the import info for the module definition.
     */
    private static ImportNodeInfo[] getImportNodeInfo(ParseTreeNode moduleDefnNode) {
        moduleDefnNode.verifyType(CALTreeParserTokenTypes.MODULE_DEFN);

        ParseTreeNode optionalCALDocNode = moduleDefnNode.firstChild();
        optionalCALDocNode.verifyType(CALTreeParserTokenTypes.OPTIONAL_CALDOC_COMMENT);

        ParseTreeNode moduleNameNode = optionalCALDocNode.nextSibling();

        ParseTreeNode importDeclarationListNode = moduleNameNode.nextSibling();
        importDeclarationListNode.verifyType(CALTreeParserTokenTypes.IMPORT_DECLARATION_LIST);

        // (List of ImportNodeInfo)
        List<ImportNodeInfo> importNodeInfoList = new ArrayList<ImportNodeInfo>();

        for (final ParseTreeNode importDeclarationNode : importDeclarationListNode) {

            importNodeInfoList.add(new ImportNodeInfo(importDeclarationNode));
        }

        return importNodeInfoList.toArray(new ImportNodeInfo[importNodeInfoList.size()]);
    }

    /**
     * Compile modules to the current program.
     *   Modules which are broken and their dependents will not be compiled.
     *   Note: the modules must not already exist in the program.
     *   Note: if a module passes parsing but fails type checking, the module will appear in the program, but contain no entities.
     *         This occurs in the type checker, which calls on the packager to create a new module in the program before type checking.
     *
     * @param moduleNames - set of names of the modules to be compiled.
     * @param sourceDefinitionMap - maps module names to the ModuleSources to compile
     * @param compiledDefinitionMap - maps module names to the existing CompiledModuleSources.
     * @param packager an object implementing the Packager interface.  This object packages and (usually) persists the target form.
     * @param foreignContextProvider Any custom foreign context provider specified by the client, or null to use the default.
     * Note that in most cases it is sensible to set this as null.
     * This is provided as a way for the Eclipse tooling to provide context not visible to the compiler.
     * @return CompilerMessage.Severity the highest error condition experienced
     */
    CompilerMessage.Severity compileModules(
            Set<ModuleName> moduleNames,
            Map<ModuleName, ModuleSourceDefinition> sourceDefinitionMap,
            Map<ModuleName, CompiledModuleSourceDefinition> compiledDefinitionMap,
            Packager packager,
            ForeignContextProvider foreignContextProvider) {

        if (DEBUG_CONCURRENT_COMPILATION) {
            // increment the total number of adjuncts compiled, and the number
            // of active threads compiling adjuncts
            synchronized (classMutex) {
                moduleCompileCount++;
                System.err.println("module compile count: " + moduleCompileCount);
                numThreadsCompiling++;
                System.err.println("Num threads: " + numThreadsCompiling);
            }

            try {
                // perform the actual compilation
                return compileModulesHelper(moduleNames, sourceDefinitionMap, compiledDefinitionMap, packager, foreignContextProvider);

            } finally {
                // this thread has finished compiling, so decrement the number of
                // active threads compiling adjuncts
                synchronized (classMutex) {
                    numThreadsCompiling--;
                }
            }
        }
        else {
            return compileModulesHelper(moduleNames, sourceDefinitionMap, compiledDefinitionMap, packager, foreignContextProvider);
        }
    }

    /**
     * Determine whether a compiled module source is valid by comparing its timestamp
     * to timestamps of the things it depends on.
     * @param compiledDefinitionInfo - the compiledDefinitionInfo for the source being checked
     * @param msd - ModuleSourceDefintion for the current module.
     * @param validCompiledDefinitionInfoMap map from module name to CompiledDefinitionInfo
     * @param importedModules - naming the modules imported by the module being checked
     * @param sourceDefinitionsInfo info about source definitions to compile
     * @return a status reporting whether if the compiled module source is up to date.
     */
    private TimestampCheckStatus isCompiledSourceUpToDate (
            CompiledDefinitionInfo compiledDefinitionInfo,
            ModuleSourceDefinition msd,
            Map<ModuleName, CompiledDefinitionInfo> validCompiledDefinitionInfoMap,
            Set<ModuleName> importedModules,
            SourceDefinitionsInfo sourceDefinitionsInfo) {

        CompiledModuleSourceDefinition cmsd = compiledDefinitionInfo.getCompiledModuleSourceDefinition();

        // If the module source is more recent than the compiled source
        // discard the compiled source.
        if (msd.getTimeStamp() > cmsd.getTimeStamp() && !sourceDefinitionsInfo.isEmptyModuleSource(msd)) {
            return new TimestampCheckStatus.OlderThanSource();
        }

        long currentSerializedTimestamp;
        try {
            currentSerializedTimestamp = compiledDefinitionInfo.getTimeStamp();
        } catch (IOException e) {
            // Log compiler error message.
            ModuleName moduleName = cmsd.getModuleName();
            logMessage(new CompilerMessage(new SourceRange(moduleName), new MessageKind.Fatal.CompilationAbortedDueToInternalModuleLoadingError(moduleName, e.getLocalizedMessage()), e));
            return new TimestampCheckStatus.ExceptionCaught();
        }

        // Now go through the list of imported modules.
        // If the compiled source for any imported module is more recent than the compiled
        // source for this module than the compiled source for this module is not valid.
        for (final ModuleName importName : importedModules) {

            CompiledDefinitionInfo importCompiledDefinitionInfo = validCompiledDefinitionInfoMap.get(importName);

            // If we can't find a compile module source for the imported module we assume we should re-compile
            // this module.
            if (importCompiledDefinitionInfo == null) {
                return new TimestampCheckStatus.DependeeNotFound(importName);
            }

            // We want to look at the timestamp actually serialized in the compiled module file.
            // This is done because of problems encountered when the timestamps on files are rounded up to
            // the nearest second by compression utilities such as winzip.
            try {
                long importSerializedTimestamp = importCompiledDefinitionInfo.getTimeStamp();

                if (importSerializedTimestamp > currentSerializedTimestamp) {
                    return new TimestampCheckStatus.OlderThanDependee(importName);
                }
            } catch (IOException e) {
                // Log compiler error message.
                ModuleName moduleName = cmsd.getModuleName();
                logMessage(new CompilerMessage(new SourceRange(moduleName), new MessageKind.Fatal.CompilationAbortedDueToInternalModuleLoadingError(moduleName, e.getLocalizedMessage()), e));
                return new TimestampCheckStatus.ExceptionCaught();
            }
        }

        return new TimestampCheckStatus.UpToDate();
    }

    /**
     * Load a module from a compiled module source.
     * Errors will be logged to the compiler's msg logger if the module failed to load.
     *
     * @param moduleName - name of the module to load
     * @param sourceDef - ModuleSourceDefinition for named module (i.e. the CAL source).
     * @param validCompiledDefinitionInfoMap - A map of module name -> CompiledDefinitionInfo for modules in the workspace with valid CompiledDefinitionInfo.
     *    (GeneratedCodeInfo is OK...).
     * @param allExistingModules - A map of ModuleName -> Module for all currently existing modules.
     * @param foreignClassLoader the classloader to use to resolve foreign classes for the module.
     * @param sourceDefinitionsInfo info about source definitions to compile
     * @return a status indicating whether an attempt was made to load the module from the compiled module source,
     * or if the compiled module source doesn't exist or is out of date.
     * @throws IOException
     */
    private CompiledModuleLoadStatus loadCompiledModule (
            ModuleName moduleName,
            ModuleSourceDefinition sourceDef,
            Map<ModuleName, CompiledDefinitionInfo> validCompiledDefinitionInfoMap,
            Map<ModuleName, Module> allExistingModules,
            ClassLoader foreignClassLoader,
            SourceDefinitionsInfo sourceDefinitionsInfo) throws IOException {

        CompiledDefinitionInfo compiledDefinitionInfo = validCompiledDefinitionInfoMap.get(moduleName);

        if (compiledDefinitionInfo == null) {
            return new CompiledModuleLoadStatus.NotFound();
        }

        CompiledModuleSourceDefinition compiledModuleSourceDefinition = compiledDefinitionInfo.getCompiledModuleSourceDefinition();

        // Do a quick check of existence and compare time stamp with the CAL source.
        // This time stamp comparison is also done by 'isCompiledSourceValid' but
        // a quick check here can save opening the input stream and reading the imported
        // modules.  The CAL source cannot be more recent than the compiled source.
        if (compiledModuleSourceDefinition.getTimeStamp() >= sourceDef.getTimeStamp() || sourceDefinitionsInfo.isEmptyModuleSource(sourceDef)) {

            int nErrorsBefore = msgLogger.getNErrors();

            // The compiled module source has passed the initial quick check of validity.
            // Now we want to open it and read its import list.  This will allow us to do
            // a deeper validity check against the modules it depends on.
            Status status = new Status("Compiled definition input stream status.");
            try {
                // Get the set of imported modules.
                // This is needed to check the validity of the current compiled module source.
                Set<ModuleName> importedModules = compiledDefinitionInfo.getImportedModuleNames();

                final TimestampCheckStatus compiledModuleTimestampCheckStatus =
                    isCompiledSourceUpToDate (compiledDefinitionInfo, sourceDef, validCompiledDefinitionInfoMap, importedModules, sourceDefinitionsInfo);

                // If the compiled source is up-to-date, try to load the module.
                // Otherwise, translate the status from isCompiledSourceUpToDate into an appropriate CompiledModuleLoadStatus to be returned.

                if (compiledModuleTimestampCheckStatus instanceof TimestampCheckStatus.UpToDate) {

                    InputStream fis = compiledModuleSourceDefinition.getInputStream(status);
                    if (fis == null) {
                        logMessage(new CompilerMessage(new SourceRange(moduleName), new MessageKind.Fatal.CompilationAbortedDueToInternalModuleLoadingError(moduleName, " Unable to access backing store.")));
                        return new CompiledModuleLoadStatus.ExceptionCaughtOrInternalError();

                    } else {
                        // Get a record import stream.
                        RecordInputStream rs = new RecordInputStream(fis);

                        try {
                            Module m = Module.load(rs, allExistingModules, foreignClassLoader, compiledDefinitionInfo.getCodeInfo(), msgLogger);

                            if (m != null) {
                                // Succeeded in loading the module.
                                allExistingModules.put (m.getName(), m);
                                packager.addModule(m);
                            }

                            // Attempted load, continue with next module
                            return new CompiledModuleLoadStatus.LoadAttempted();

                        } finally {
                            rs.close();
                        }
                    }

                } else if (compiledModuleTimestampCheckStatus instanceof TimestampCheckStatus.OlderThanSource) {
                    return new CompiledModuleLoadStatus.OlderThanSource();

                } else if (compiledModuleTimestampCheckStatus instanceof TimestampCheckStatus.DependeeNotFound) {
                    final ModuleName dependeeName = ((TimestampCheckStatus.DependeeNotFound)compiledModuleTimestampCheckStatus).getDependeeName();
                    return new CompiledModuleLoadStatus.DependeeNotFound(dependeeName);

                } else if (compiledModuleTimestampCheckStatus instanceof TimestampCheckStatus.OlderThanDependee) {
                    final ModuleName dependeeName = ((TimestampCheckStatus.OlderThanDependee)compiledModuleTimestampCheckStatus).getDependeeName();
                    return new CompiledModuleLoadStatus.OlderThanDependee(dependeeName);

                } else if (compiledModuleTimestampCheckStatus instanceof TimestampCheckStatus.ExceptionCaught) {
                    return new CompiledModuleLoadStatus.ExceptionCaughtOrInternalError();

                } else {
                    // unknown kind of TimestampCheckStatus
                    throw new IllegalStateException("Unknown TimestampCheckStatus: " + compiledModuleTimestampCheckStatus);
                }

            } catch (Exception e) {

                if (msgLogger.getNErrors() > nErrorsBefore || e instanceof UnableToResolveForeignEntityException) {

                    // If the exception is an UnableToResolveForeignEntityException, there is
                    // a CompilerMessage inside that we should be logging.
                    if (e instanceof UnableToResolveForeignEntityException) {
                        try {
                            logMessage(((UnableToResolveForeignEntityException)e).getCompilerMessage());
                        } catch (AbortCompilation ace) {
                            // Yeah, yeah, we know
                            //logMessage can throw a AbortCompilation if a FATAL message was sent.
                        }
                    }

                    //if an error occurred previously, we continue to compile the program to try to report additional
                    //meaningful compilation errors. However, this can produce spurious exceptions related to the fact
                    //that the program state does not satisfy preconditions because of the initial error(s). We don't
                    //report the spurious exception as an internal coding error.
                    logMessage(new CompilerMessage(new SourceRange(moduleName), new MessageKind.Info.UnableToRecover()));
                } else {

                    //"Compilation aborted due to an internal error in loading the compiled module {0}. Please contact Business Objects. Detail: {1}"
                    logMessage(
                            new CompilerMessage(new SourceRange(moduleName),
                                    new MessageKind.Fatal.CompilationAbortedDueToInternalModuleLoadingError(moduleName, e.getLocalizedMessage()), e));
                }

                return new CompiledModuleLoadStatus.ExceptionCaughtOrInternalError();
            }

        } else {
            return new CompiledModuleLoadStatus.OlderThanSource();
        }
    }

    /**
     * Compile modules to the current program.
     *   Modules which are broken and their dependents will not be compiled.
     *   Note: the modules must not already exist in the program.
     *   Note: if a module passes parsing but fails type checking, the module will appear in the program, but contain no entities.
     *         This occurs in the type checker, which calls on the packager to create a new module in the program before type checking.
     *
     * @param moduleNames - set of names of the modules to be compiled.
     * @param sourceDefinitionMap (ModuleName -> ModuleSourceDefinition) maps module names to the ModuleSources to compile
     * @param compiledDefinitionMap (ModuleName -> CompiledModuleSourceDefinition) maps module names to the CompiledModuleSources to load.
     * @param packager an object implementing the Packager interface.  This object packages and (usually) persists the target form.
     * @param foreignContextProvider Any custom foreign context provider specified by the client, or null to use the default.
     * @return CompilerMessage.Severity the highest error condition experienced
     */
    private CompilerMessage.Severity compileModulesHelper(
            Set<ModuleName> moduleNames,
            Map<ModuleName, ModuleSourceDefinition> sourceDefinitionMap,
            Map<ModuleName, CompiledModuleSourceDefinition> compiledDefinitionMap,
            Packager packager,
            ForeignContextProvider foreignContextProvider) {

        if (sourceDefinitionMap == null || packager == null || compiledDefinitionMap == null) {
            throw new NullPointerException();
        }

        // Short circuit if there are no modules to compile.
        // This can happen for instance if we try to compile dirty modules only, but find that there are no dirty modules.
        if (moduleNames.isEmpty()) {
            return CompilerMessage.Severity.INFO;
        }

        SourceDefinitionsInfo sourceDefinitionsInfo = new SourceDefinitionsInfo(sourceDefinitionMap);

        // Store current packager
        this.packager = packager;
        if (packager == null) {
            throw new IllegalArgumentException("CALCompiler.compile: must have a packager.");
        }

        // Replace the logger.
        CompilerMessageLogger oldLogger = msgLogger;
        CompilerMessageLogger compileLogger = new MessageLogger(false);  // Do not abort compilation with this logger.
        msgLogger = compileLogger;

        try {

            /* the names of modules with errors found during dependency ordering analysis. */
            Set<ModuleName> preparseBrokenModuleNameSet = new HashSet<ModuleName>();

            /*
             * In order to determine dependency ordering, it is necessary to know, for a given source definition,
             * the name of the module being represented, plus the modules which it imports.
             *
             * Previously, this was done by fully parsing the source definitions, and analyzing the parse trees for this info.
             * The source model for the module was also generated, since the data for this was readily available.
             * However, this led to memory-use issues, as these tree structures are potentially very bulky objects.
             * Moreover, they are expensive to construct, meaning that the work done during this initial compilation phase
             * could not be done in parallel with other compilation work.
             *
             * Now, before any real parsing is done, all source definitions are only parsed as far as required in order to determine,
             * for a given source definition, the name of the module and its imports.  The dependency ordering is calculated
             * from this info, and using this ordering, modules are compiled (including parsed) in dependency order.
             */

            // Map of ModuleName -> Module which maps module names to modules for all existing/compiled/loaded modules.
            Map<ModuleName, Module> allExistingModules = new HashMap<ModuleName, Module>();

            //
            // Determine dependency ordering.
            //
            // Gather the names of already packaged modules.
            Set<ModuleName> packagedModuleNamesSet = new HashSet<ModuleName>();
            for (final Module m : packager.getProgram().getModules()) {
                ModuleName moduleName = m.getName();
                packagedModuleNamesSet.add(moduleName);
                allExistingModules.put (moduleName, m);
            }

            /*
             * Map from module name to the CompiledDefinitionInfo for that module,
             *   for those definitions which don't need to be regenerated.
             *
             * This map starts out mapping those modules which contain CompiledDefinitionInfo for those modules for which
             *   GeneratedCodeInfo.needToRegenerate() returns false.
             * In the compile loop below, further pruning occurs -- a module is removed from the map if it is not loaded.
             */
            Map<ModuleName, CompiledDefinitionInfo> validCompiledDefinitionInfoMap = getValidCodeInfoCompiledDefinitionInfoMap(compiledDefinitionMap);

            // The module dependency graph resulting from this phase.
            SourceDefinitionsHeaderParseInfo parseInfo =
                calculateParseInfo(sourceDefinitionsInfo, validCompiledDefinitionInfoMap, packagedModuleNamesSet, preparseBrokenModuleNameSet);

            // Figure out the order in which the modules should be compiled.
            Graph<ModuleName> moduleDependencyGraph = getModuleDependencyGraph(parseInfo, packagedModuleNamesSet, preparseBrokenModuleNameSet);

            ModuleName[] moduleNamesInCompileOrder = getModuleDependencyOrder(moduleDependencyGraph, parseInfo, preparseBrokenModuleNameSet);

            // Calculate the names of broken modules and their dependents (direct or indirect).
            Set<ModuleName> dependentBrokenModulesSet = new HashSet<ModuleName>();
            for (final ModuleName brokenParseModuleName : preparseBrokenModuleNameSet) {
                addDependentModulesToSet(moduleDependencyGraph, brokenParseModuleName, dependentBrokenModulesSet);
            }

            //
            // Compile modules in dependency order.
            //
            for (final ModuleName moduleName : moduleNamesInCompileOrder) {
                // skip if dependent on a broken module
                if (dependentBrokenModulesSet.contains(moduleName)) {
                    continue;
                }

                // skip if included in graph just because another module depended on it
                // (ie: it is not an actual module, but was included for dependency analysis only)
                if (packagedModuleNamesSet.contains(moduleName)) {
                    continue;
                }

                // replace the compiler's message logger with one to track messages for this module.
                CompilerMessageLogger compileModuleLogger = msgLogger;
                msgLogger = new MessageLogger(true);

                try {
                    ModuleSourceDefinition sourceDef = sourceDefinitionsInfo.getDefinition(moduleName);

                    // Get the classloader to use to resolve foreign classes for the module.
                    // If there is no foreign context provider, use the classloader for this class (a reasonable default).
                    // For now it is an error if the provided provider does not provide a classloader,
                    ClassLoader foreignClassLoader = null;
                    if (foreignContextProvider != null) {
                        foreignClassLoader = foreignContextProvider.getClassLoader(moduleName);
                        if (foreignClassLoader == null) {
                            msgLogger.logMessage(new CompilerMessage(new SourceRange(moduleName), new MessageKind.Error.NoClassLoaderProvided(moduleName)));
                            addDependentModulesToSet(moduleDependencyGraph, moduleName, dependentBrokenModulesSet);
                            continue;
                        }

                    } else {
                        foreignClassLoader = getClass().getClassLoader();
                    }

                    //
                    // Attempt to load.
                    //
                    int nErrorsBeforeLoad = msgLogger.getNErrors();
                    final CompiledModuleLoadStatus compiledModuleLoadStatus = loadCompiledModule (moduleName, sourceDef, validCompiledDefinitionInfoMap, allExistingModules, foreignClassLoader, sourceDefinitionsInfo);

                    if (compiledModuleLoadStatus instanceof CompiledModuleLoadStatus.LoadAttempted) {
                        // We've loaded the current module from a compiled source.

                        // Check if there was a problem loading the compiled module.
                        if (msgLogger.getNErrors() != nErrorsBeforeLoad) {
                            // Add this module and dependents to the set of broken modules.
                            addDependentModulesToSet(moduleDependencyGraph, moduleName, dependentBrokenModulesSet);
                        }

                        // Continue with the next module.
                        continue;
                    }

                    // If the module source is empty, the module is a sourceless module.
                    // If we're here, the compiled source has failed to load, so this is a broken module.
                    if (sourceDefinitionsInfo.isEmptyModuleSource(sourceDef)) {

                        // We attempt to translate the status returned by loadCompiledModule into a helpful Error message
                        SourceRange sourceRange = new SourceRange(moduleName);

                        if (compiledModuleLoadStatus instanceof CompiledModuleLoadStatus.NotFound) {
                            msgLogger.logMessage(new CompilerMessage(sourceRange,
                                new MessageKind.Error.CannotLoadSourcelessModuleMaybeInvalidCar(moduleName)));

                        } else if (compiledModuleLoadStatus instanceof CompiledModuleLoadStatus.OlderThanSource) {
                            msgLogger.logMessage(new CompilerMessage(sourceRange,
                                new MessageKind.Error.CannotLoadSourcelessModuleMaybeInvalidCar(moduleName)));

                        } else if (compiledModuleLoadStatus instanceof CompiledModuleLoadStatus.DependeeNotFound) {
                            final ModuleName dependeeName = ((CompiledModuleLoadStatus.DependeeNotFound)compiledModuleLoadStatus).getDependeeName();
                            msgLogger.logMessage(new CompilerMessage(sourceRange,
                                new MessageKind.Error.CannotLoadSourcelessModuleDependeeNotFound(moduleName, dependeeName)));

                        } else if (compiledModuleLoadStatus instanceof CompiledModuleLoadStatus.OlderThanDependee) {
                            final ModuleName dependeeName = ((CompiledModuleLoadStatus.OlderThanDependee)compiledModuleLoadStatus).getDependeeName();
                            msgLogger.logMessage(new CompilerMessage(sourceRange,
                                new MessageKind.Error.CannotLoadSourcelessModuleOlderThanDependee(moduleName, dependeeName)));

                        } else {
                            // there are other kinds of statuses, e.g. LoadAttempted, ExceptionCaughtOrInternalError
                            // in these cases we give the generic error message because there aren't more details.
                            msgLogger.logMessage(new CompilerMessage(sourceRange,
                                new MessageKind.Error.InvalidSourcelessModule(moduleName)));
                        }

                        addDependentModulesToSet(moduleDependencyGraph, moduleName, dependentBrokenModulesSet);
                        continue;
                    }

                    //
                    // Attempt to compile.
                    //

                    boolean moduleCompiledSuccessfully = compileModuleInternal(sourceDef, foreignClassLoader);
                    if (!moduleCompiledSuccessfully) {
                        // if compilation fails, add this to the list of broken modules
                        addDependentModulesToSet(moduleDependencyGraph, moduleName, dependentBrokenModulesSet);
                        continue;
                    }

                    // Add the module to the set of existing modules.
                    Module currentModule = packager.getCurrentModule();
                    allExistingModules.put (currentModule.getName(), currentModule);

                    // If we're here, the module shouldn't be loaded, which means that any previous CompiledDefinitionInfo for it will be out of date
                    // Note that because we compile modules in dependency order, modules which depend on this one will also be found to need regeneration.
                    validCompiledDefinitionInfoMap.remove(moduleName);

                } catch (CompilerMessage.AbortCompilation e) {
                    // Compilation aborted

                    // Make sure that any dependents are added to the set of things that can't be compiled.
                    addDependentModulesToSet(moduleDependencyGraph, moduleName, dependentBrokenModulesSet);

                } catch (Exception e) {
                    try {
                        if (msgLogger.getNErrors() > 0 || e instanceof UnableToResolveForeignEntityException) {

                            // If the exception is an UnableToResolveForeignEntityException, there is
                            // a CompilerMessage inside that we should be logging.
                            if (e instanceof UnableToResolveForeignEntityException) {
                                try {
                                    logMessage(((UnableToResolveForeignEntityException)e).getCompilerMessage());
                                } catch (AbortCompilation ace) {
                                    // Yeah, yeah, we know
                                    //logMessage can throw a AbortCompilation if a FATAL message was sent.
                                }
                            }

                            //if an error occurred previously while compiling this module, we continue to compile the module to
                            //try to report additional meaningful compilation errors. However, this can produce spurious exceptions
                            //related to the fact that the module state does not satisfy preconditions because of the initial error(s).
                            //We don't report the spurious exception as an internal coding error.
                            logMessage(new CompilerMessage(new SourceRange(moduleName), new MessageKind.Info.UnableToRecover()));

                            // Make sure that any dependents are added to the set of things that can't be compiled.
                            addDependentModulesToSet(moduleDependencyGraph, moduleName, dependentBrokenModulesSet);

                        } else {
                            // Major failure - internal coding error
                            logMessage(new CompilerMessage(new MessageKind.Fatal.InternalCodingError(), e));
                        }
                    } catch (AbortCompilation ace) {
                        // Yeah, yeah, we know
                        //logMessage can throw a AbortCompilation if a FATAL message was sent.
                    }

                } finally {
                    // now replace back the compiler's message logger, after adding all the messages from this module.
                    try {
                        compileModuleLogger.logMessages(msgLogger);
                    } finally {
                        msgLogger = compileModuleLogger;
                    }
                }
            }

        } catch (Exception e) {
            // Catch exceptions which occur during dependency analysis etc.
            try {
                if (msgLogger.getNErrors() > 0 || e instanceof UnableToResolveForeignEntityException) {

                    // If the exception is an UnableToResolveForeignEntityException, there is
                    // a CompilerMessage inside that we should be logging.
                    if (e instanceof UnableToResolveForeignEntityException) {
                        try {
                            logMessage(((UnableToResolveForeignEntityException)e).getCompilerMessage());
                        } catch (AbortCompilation ace) {
                            // Yeah, yeah, we know
                            //logMessage can throw a AbortCompilation if a FATAL message was sent.
                        }
                    }

                    //if an error occurred previously, we continue to try to report additional meaningful compilation errors.
                    //However, this can produce spurious exceptions related to the fact that the program state does not
                    //satisfy preconditions because of the initial error(s).
                    //We don't report the spurious exception as an internal coding error.
                    logMessage(new CompilerMessage(new MessageKind.Info.UnableToRecover()));

                } else {
                    // Major failure - internal coding error
                    logMessage(new CompilerMessage(new MessageKind.Fatal.InternalCodingError(), e));
                }
            } catch (AbortCompilation ace) {
                // Yeah, yeah, we know
                //logMessage can throw a AbortCompilation if a FATAL message was sent.
            }

        } finally {
            msgLogger = oldLogger;

            try {
                msgLogger.logMessages(compileLogger);
            } catch (CompilerMessage.AbortCompilation e) {
                // The ol' logger has got too many errors.
            }
        }

        //close the packager if there have been no errors otherwise abort
        try {
            if (compileLogger.getNErrors() == 0) {
                packager.close(msgLogger);
            } else {
                packager.abort(msgLogger);
            }

        } catch (Packager.PackagerException e) {
            try {
                ModuleName moduleName = packager.getCurrentModule().getName();
                logMessage(new CompilerMessage(new SourceRange(moduleName), new MessageKind.Error.UnableToCloseModuleAndPackage(e.toString()) ));
            } catch (AbortCompilation ace) {
            }
        }

        return compileLogger.getMaxSeverity();
    }

    /**
     * Adds the dependent modules of the given module to a set.
     * @param moduleDependencyGraph the graph.
     * @param moduleName the module whose dependents are to be added to the set.
     * @param moduleNameSet the set to be modified.
     */
    private static void addDependentModulesToSet(Graph<ModuleName> moduleDependencyGraph, ModuleName moduleName, Set<ModuleName> moduleNameSet) {
        Set<ModuleName> dependentVertexNames = moduleDependencyGraph.getDependentVertexNames(moduleName);
        moduleNameSet.addAll(dependentVertexNames);
    }

    /**
     * Internal method to handle the non-load part of compilation of a single module.
     * Compiler messages are logged to the current message logger.  This logger should not contain any errors when this method is called.
     *
     * @param sourceDef the source definition of the module to compile.
     * @param foreignClassLoader the classloader to use to resolve foreign classes for the module.
     * @return whether the module were compiled without errors.
     * @throws Packager.PackagerException thrown if there is a problem wrapping the module
     * @throws CompilerMessage.AbortCompilation thrown if there is a problem when logging messages.
     */
    private boolean compileModuleInternal(ModuleSourceDefinition sourceDef, ClassLoader foreignClassLoader)
            throws Packager.PackagerException, CompilerMessage.AbortCompilation, UnableToResolveForeignEntityException {

        ModuleName moduleName = sourceDef.getModuleName();

        this.currentModuleTimeStamp = sourceDef.getTimeStamp();

        int nErrorsBefore = msgLogger.getNErrors();

        //
        // Parse the module's source definition.
        //
        Pair<ParseTreeNode, SourceModel.ModuleDefn> parseResultPair = parseModule(sourceDef);

        // Check for parse failure.
        if (msgLogger.getNErrors() > nErrorsBefore) {
            return false;
        }

        // The parse tree node.
        ParseTreeNode moduleDefnNode = parseResultPair.fst();

        // The SourceModel (for metrics processing) - get this before the typechecker can change the parse tree
        SourceModel.ModuleDefn moduleDefnSourceModel = parseResultPair.snd();

        //
        // First, scan the module for @deprecated blocks.
        //
        getDeprecationScanner().processModule(moduleDefnSourceModel);

        //
        // Type check the module.
        //
        typeChecker.checkModule(moduleDefnNode, foreignClassLoader);

        // Check for type check failure.
        if (msgLogger.getNErrors() > nErrorsBefore) {
            return false;
        }

        //
        // Generate the raw and module-level source metric data for this module.
        //
        ModuleTypeInfo moduleTypeInfo = packager.getModuleTypeInfo(moduleName);
        if (moduleDefnSourceModel != null) {
            SourceMetricFinder.updateRawMetricData(moduleDefnSourceModel, moduleTypeInfo);
            moduleTypeInfo.setModuleSourceMetrics(new ModuleSourceMetrics(moduleTypeInfo));
        }

        //
        // Convert the antlr-generated AST to our own internal machine-specific code.
        //
        ExpressionGenerator expressionGenerator = new ExpressionGenerator(this, moduleTypeInfo, false);
        ParseTreeNode outerDefnListNode = moduleDefnNode.getChild(4);
        outerDefnListNode.verifyType(CALTreeParserTokenTypes.OUTER_DEFN_LIST);
        expressionGenerator.generateCode(outerDefnListNode);
        this.currentModuleTimeStamp = 0L;

        //
        // Notify the packager that the module is done.
        //
        packager.wrapModule(msgLogger);     // throws PackagerException..

        // Return whether the module compiled successfully.
        return (msgLogger.getNErrors() == nErrorsBefore);
    }

    /**
     * @param compiledDefinitionMap (ModuleName->CompiledModuleSourceDefinition) map from module name to the compiled module source for that module.
     * @return (ModuleName->CompiledDefinitionInfo) Map from module name to the CompiledDefinitionInfo for that module.
     *   This map will only contain entries for which the GeneratedCodeInfo does not indicate that the code needs to be regenerated.
     */
    private Map<ModuleName, CompiledDefinitionInfo> getValidCodeInfoCompiledDefinitionInfoMap(
            Map<ModuleName, CompiledModuleSourceDefinition> compiledDefinitionMap) {

        Map<ModuleName, CompiledDefinitionInfo> result = new HashMap<ModuleName, CompiledDefinitionInfo>();

        for (final Map.Entry<ModuleName, CompiledModuleSourceDefinition> entry : compiledDefinitionMap.entrySet()) {

            ModuleName moduleName = entry.getKey();
            CompiledModuleSourceDefinition cmsd = entry.getValue();
            CompiledDefinitionInfo info = new CompiledDefinitionInfo(cmsd);

            GeneratedCodeInfo codeInfo;
            try {
                codeInfo = info.getCodeInfo();
            } catch (IOException e) {
                // couldn't get the code info.
                continue;
            }

            // Check whether the compiled definition needs to be regenerated.
            if (codeInfo.needToRegenerate()) {
                continue;
            }

            result.put(moduleName, info);
        }

        return result;
    }

    /**
     * Determine the dependency ordering for group of module definitions.
     * Avoid generating and keeping parse trees here, as these are potentially very bulky objects.
     *
     * @param sourceDefinitionsInfo info for the source definitions to compile.
     * @param validCompiledDefinitionInfoMap - String -> CompiledDefinitionInfo
     * @param packagedModuleNamesSet the names of already packaged modules
     * @param preparseBrokenModuleNameSet (Set of String) the names of modules with errors during dependency ordering analysis.
     *   This collection will be populated by this method.
     * @return the parse info for the source definitions.
     * Note that parse info will be returned for all source definitions for which this can be determined.
     * These source definitions may be broken for other reasons, for instance if the source definition name does
     * not match the name parsed out of the definition.
     */
    private SourceDefinitionsHeaderParseInfo calculateParseInfo(
            SourceDefinitionsInfo sourceDefinitionsInfo,
            Map<ModuleName, CompiledDefinitionInfo> validCompiledDefinitionInfoMap,
            Set<ModuleName> packagedModuleNamesSet,
            Set<ModuleName> preparseBrokenModuleNameSet) {

        Set<ModuleName> nonParseableModuleNamesSet = new HashSet<ModuleName>();

        /* (ModuleName->ImportNodeInfo[]) Map from module name to its import info. */
        // In order to obtain deterministic ordering of modules in the dependency graph, use a sorted map.
        SortedMap<ModuleName, ImportNodeInfo[]> moduleNameToImportNodeInfoMap = new TreeMap<ModuleName, ImportNodeInfo[]>();

        /* (ModuleName->ParseTreeNode) Map from module name to its parse tree node, if any.
         * For source positions while error checking. */
        Map<ModuleName, ParseTreeNode> moduleNameToHeaderDefnNodeMap = new HashMap<ModuleName, ParseTreeNode>();

        // Iterate over the source definitions to get the dependency ordering.
        for (final ModuleSourceDefinition sourceDef : sourceDefinitionsInfo.getSourceDefinitions()) {

            ModuleName moduleSourceName = sourceDef.getModuleName();

            // replace the compiler's message logger with one to track messages for this module.
            CompilerMessageLogger compilerLogger = msgLogger;
            msgLogger = new MessageLogger();

            try {
                this.currentModuleTimeStamp = 0L;

                // In the case of a source definition that is defined in terms of a SourceModel
                // object, we can use the toParseTreeNode method to completely bypass the parser
                // and get a parse tree directly. This should be a considerable efficiency gain.
                // We also associate the SourceModel itself with the module name, so that we can
                // avoid having to generate a redundant SourceModel when it's time to compute metrics
                if (sourceDef instanceof SourceModelModuleSource) {
                    if (moduleNameToImportNodeInfoMap.put(moduleSourceName, getImportNodeInfo(((SourceModelModuleSource)sourceDef).getModuleDefn())) != null) {
                        logMessage(new CompilerMessage(new SourceRange(moduleSourceName), new MessageKind.Error.AttemptedRedefinitionOfModule(moduleSourceName)));
                    }

                } else {

                    // If the compiled module source definition is valid we can use it.
                    // We consider the compiled source valid if the CAL source is not
                    // more recent.
                    CompiledDefinitionInfo compiledDefinitionInfo = validCompiledDefinitionInfoMap.get(moduleSourceName);
                    boolean compiledDefinitionUpToDate = false;

                    boolean loadedImports = false;
                    if (compiledDefinitionInfo != null) {
                        CompiledModuleSourceDefinition compiledModuleSourceDefinition = compiledDefinitionInfo.getCompiledModuleSourceDefinition();

                        if (sourceDef.getTimeStamp() <= compiledModuleSourceDefinition.getTimeStamp()) {
                            // Read the imports from the compiled module information.
                            Set<ModuleName> imports = null;
                            try {
                                imports = compiledDefinitionInfo.getImportedModuleNames();
                            } catch (IOException e) {
                                logMessage(new CompilerMessage(new SourceRange(moduleSourceName),
                                                               new MessageKind.Warning.DebugMessage("Failed reading imports from compiled module info for " + moduleSourceName), e));
                            }

                            if (imports != null) {
                                loadedImports = true;
                                ImportNodeInfo importInfo[] = new ImportNodeInfo [imports.size()];
                                int index = 0;
                                for (final ModuleName importedModule : imports) {
                                    importInfo[index++] = new ImportNodeInfo (importedModule);
                                }

                                // Check whether this module has already been defined.
                                if (moduleNameToImportNodeInfoMap.put(compiledModuleSourceDefinition.getModuleName(), importInfo) != null) {
                                    logMessage(new CompilerMessage(new SourceRange(compiledModuleSourceDefinition.getModuleName()),
                                                                   new MessageKind.Error.AttemptedRedefinitionOfModule(compiledModuleSourceDefinition.getModuleName())));
                                }
                            }

                            compiledDefinitionUpToDate = true;
                        }
                    }

                    if (!loadedImports) {
                        // Was unable to load imports from compiled module definition parse them out
                        // of the module source.

                        Reader moduleReader = sourceDef.getSourceReader(new Status("Read module status."));
                        if (moduleReader == null) {
                            logMessage(new CompilerMessage(new SourceRange(moduleSourceName),
                                                           new MessageKind.Error.CouldNotReadModuleSource(moduleSourceName)));
                            continue;
                        }

                        boolean isEmptyModuleSource = sourceDefinitionsInfo.isEmptyModuleSource(sourceDef);
                        if (isEmptyModuleSource) {
                            SourceRange sourceRange = new SourceRange(moduleSourceName);
                            // We will try to be friendly and report an appropriate error about the sourceless module if possible
                            if (compiledDefinitionInfo == null) {
                                // a sourceless module should have a CMI file, but it's not found - maybe the Car is broken
                                logMessage(new CompilerMessage(sourceRange, new MessageKind.Error.CannotLoadSourcelessModuleMaybeInvalidCar(moduleSourceName)));
                            } else if (!compiledDefinitionUpToDate) {
                                // the CMI file for a sourceless module should be newer than the empty stub CAL file - maybe the Car is broken
                                logMessage(new CompilerMessage(sourceRange, new MessageKind.Error.CannotLoadSourcelessModuleMaybeInvalidCar(moduleSourceName)));
                            } else {
                                // the problem is somewhere else, so we report the general error
                                logMessage(new CompilerMessage(sourceRange, new MessageKind.Error.InvalidSourcelessModule(moduleSourceName)));
                            }
                            continue;
                        }

                        moduleReader = new BufferedReader(moduleReader);

                        setInputStream(moduleReader, moduleSourceName);

                        ModuleName moduleName = null;
                        try {
                            int nOldErrors = msgLogger.getNErrors();

                            // Call the parser to parse a module header definition
                            parser.moduleHeader();

                            if (msgLogger.getNErrors() == nOldErrors) {

                                ParseTreeNode moduleHeaderDefnNode = (ParseTreeNode)parser.getAST();

                                // sanity check
                                moduleHeaderDefnNode.verifyType(CALTreeParserTokenTypes.MODULE_DEFN);

                                ParseTreeNode optionalCALDocNode = moduleHeaderDefnNode.firstChild();
                                optionalCALDocNode.verifyType(CALTreeParserTokenTypes.OPTIONAL_CALDOC_COMMENT);

                                ParseTreeNode moduleNameNode = optionalCALDocNode.nextSibling();
                                moduleName = ModuleNameUtilities.getModuleNameFromParseTree(moduleNameNode);

                                moduleNameToHeaderDefnNodeMap.put(moduleName, moduleHeaderDefnNode);

                                // check that module name corresponds to source name
                                if (moduleSourceName != null && !moduleSourceName.equals(moduleName)) {
                                    logMessage(new CompilerMessage(moduleNameNode, new MessageKind.Error.ModuleNameDoesNotCorrespondToSourceName(moduleName, moduleSourceName)));
                                }

                                // Check whether this module has already been defined.
                                if (moduleNameToImportNodeInfoMap.put(moduleName, getImportNodeInfo(moduleHeaderDefnNode)) != null) {
                                    logMessage(new CompilerMessage(moduleNameNode, new MessageKind.Error.AttemptedRedefinitionOfModule(moduleName)));
                                }
                            }

                        } catch (antlr.RecognitionException e) {
                            // Recognition (syntax) error
                            final SourceRange sourceRange = CALParser.makeSourceRangeFromException(e);
                            logMessage(new CompilerMessage(sourceRange, new MessageKind.Error.SyntaxError(), e));

                        } catch (antlr.TokenStreamException e) {
                            // Bad token stream.  Maybe a bad token (eg. a stray "$" sign)
                            logMessage(new CompilerMessage(new SourceRange(moduleSourceName), new MessageKind.Error.BadTokenStream(), e));

                        } finally {
                            // close the reader if any.
                            if (moduleReader != null) {
                                try {
                                    moduleReader.close();
                                } catch (IOException ioe) {
                                    COMPILER_LOGGER.log(Level.FINE, "Problem closing file for module \"" + moduleSourceName + "\"");
                                }
                            }

                            // track whether the parse failed.
                            if (msgLogger.getMaxSeverity().compareTo (CompilerMessage.Severity.ERROR) >= 0) {
                                if (moduleName == null) {
                                    // parsing to valid AST failed since the name node was not found;
                                    // so use the name specified by the caller
                                    moduleName = moduleSourceName;
                                    nonParseableModuleNamesSet.add(moduleName);
                                }
                                preparseBrokenModuleNameSet.add(moduleName);
                            }
                        }
                    }
                }

            } finally {
                // now replace back the compiler's message logger, after adding all the messages from this module.
                try {
                    compilerLogger.logMessages(msgLogger);
                } catch (CompilerMessage.AbortCompilation e) {
                    // Don't stop if fatal errors are copied over
                } finally {
                    msgLogger = compilerLogger;
                }
            }
        }

        return new SourceDefinitionsHeaderParseInfo(nonParseableModuleNamesSet, moduleNameToImportNodeInfoMap, moduleNameToHeaderDefnNodeMap);
    }

    /**
     * Parse a module from its source definition.
     * If there are problems parsing the module, errors will be logged to the compiler's message logger.
     *
     * @param sourceDef the module's source definition.
     * @return Pair (ParseTreeNode, SourceModel.ModuleDefn) the resulting parse tree node for the module, and its source model.
     * Null if the module source could not be read (in which case a compiler error will be logged).
     * If parsing failed, one or both elements of the pair may be null.
     */
    private Pair<ParseTreeNode, SourceModel.ModuleDefn> parseModule(ModuleSourceDefinition sourceDef) {
        // The name of the module.
        ModuleName moduleSourceName = sourceDef.getModuleName();

        // The parse tree node.
        ParseTreeNode moduleDefnNode = null;

        // The SourceModel (for metrics processing) - get this before the typechecker can change the parse tree
        SourceModel.ModuleDefn moduleDefnSourceModel = null;

        int nOldErrors = msgLogger.getNErrors();

        // In the case of a source definition that is defined in terms of a SourceModel
        // object, we can use the toParseTreeNode method to completely bypass the parser
        // and get a parse tree directly. This should be a considerable efficiency gain.
        // We also associate the SourceModel itself with the module name, so that we can
        // avoid having to generate a redundant SourceModel when it's time to compute metrics
        if (sourceDef instanceof SourceModelModuleSource) {
            moduleDefnNode = ((SourceModelModuleSource)sourceDef).getParseTreeNode();
            moduleDefnSourceModel = ((SourceModelModuleSource)sourceDef).getModuleDefn();

        } else {

            Reader moduleReader = sourceDef.getSourceReader(new Status("Read module status."));
            if (moduleReader == null) {
                logMessage(new CompilerMessage(new SourceRange(moduleSourceName), new MessageKind.Error.CouldNotReadModuleSource(moduleSourceName)));
                return null;
            }
            moduleReader = new BufferedReader(moduleReader);

            setInputStream(moduleReader, moduleSourceName);

            try {
                // Call the parser to parse the module definition
                parser.module();

                moduleDefnNode = (ParseTreeNode)parser.getAST();

            } catch (antlr.RecognitionException e) {
                // Recognition (syntax) error
                final SourceRange sourceRange = CALParser.makeSourceRangeFromException(e);
                logMessage(new CompilerMessage(sourceRange, new MessageKind.Error.SyntaxError(), e));

            } catch (antlr.TokenStreamException e) {
                // Bad token stream.  Maybe a bad token (eg. a stray "$" sign)
                logMessage(new CompilerMessage(new SourceRange(moduleSourceName), new MessageKind.Error.BadTokenStream(), e));

            } finally {
                // close the reader if any.
                if (moduleReader != null) {
                    try {
                        moduleReader.close();
                    } catch (IOException ioe) {
                        COMPILER_LOGGER.log(Level.FINE, "Problem closing file for module \"" + moduleSourceName + "\"");
                    }
                }
            }

            // Only try to create a source model if the parse was successful
            if (moduleDefnNode != null && msgLogger.getNErrors() == nOldErrors) {
                moduleDefnSourceModel = SourceModelBuilder.buildModuleDefn(moduleDefnNode);
            }
        }

        // If parsing succeeded to produce an AST, continue with this module node..
        // Note: only invoke the tree parser if there are no errors when parsing the module.
        //   Otherwise the tree parser is sure to fail (in a fatal error) and so there is no point doing this.
        if (moduleDefnNode != null && msgLogger.getNErrors() == nOldErrors) {

            // Debug..

            if (DEBUG_SOURCE_MODEL || inUnitTestDebugSourceModelMode) {
                //use the generated ParseTree to create a SourceModel.ModuleDefn.
                SourceModel.ModuleDefn moduleDefn = SourceModelBuilder.buildModuleDefn(moduleDefnNode);

                if (DEBUG_SOURCE_MODEL_VISITOR || inUnitTestDebugSourceModelMode) {
                    SourceModel.ModuleDefn newModuleDefn = (SourceModel.ModuleDefn)moduleDefn.accept(new SourceModelCopier<Void>(), null);

                    moduleDefn = newModuleDefn;
                    moduleDefn.accept(new SourceModelTraverser<Void, Void>(), null);
                }

                if (DEBUG_SOURCE_MODEL_TEXT || inUnitTestDebugSourceModelMode) {
                    // ...then create module text from the source model. It should work when parsed again
                    // i.e. the source model should represent the same CAL as the original CAL text.

                    ModuleSourceDefinition sourceBuilderModuleSource = new SourceModelModuleSource(moduleDefn);
                    Reader sourceBuilderModuleReader = sourceBuilderModuleSource.getSourceReader(new Status("Read module status."));
                    if (sourceBuilderModuleReader == null) {
                        logMessage(new CompilerMessage(new SourceRange(moduleSourceName), new MessageKind.Error.CouldNotReadModuleSource(moduleSourceName)));
                        return null;
                    }
                    sourceBuilderModuleReader = new BufferedReader(sourceBuilderModuleReader);

                    setInputStream(sourceBuilderModuleReader, moduleSourceName);

                    try {
                        parser.module();
                        moduleDefnNode = (ParseTreeNode)parser.getAST();
                    } catch (antlr.RecognitionException e) {
                        // Recognition (syntax) error
                        final SourceRange sourceRange = CALParser.makeSourceRangeFromException(e);
                        logMessage(new CompilerMessage(sourceRange, new MessageKind.Error.SyntaxError(), e));

                    } catch (antlr.TokenStreamException e) {
                        // Bad token stream.  Maybe a bad token (eg. a stray "$" sign)
                        logMessage(new CompilerMessage(new SourceRange(moduleSourceName), new MessageKind.Error.BadTokenStream(), e));

                    }

                } else {
                    moduleDefnNode = moduleDefn.toParseTreeNode();
                }
            }

            // End debug..

            //
            //Walk the parse tree as a sanity check on the generated AST and of the tree parser
            //
            try {
                treeParser.module(moduleDefnNode);

            } catch (antlr.RecognitionException e) {
                // Recognition (syntax) error
                final SourceRange sourceRange = CALParser.makeSourceRangeFromException(e);
                logMessage(new CompilerMessage(sourceRange, new MessageKind.Error.SyntaxError(), e));
            }
        }

        return new Pair<ParseTreeNode, SourceModel.ModuleDefn>(moduleDefnNode, moduleDefnSourceModel);
    }

    /**
     * Compile an adjunct from an AdjunctSource.
     * @param adjunctSource the adjunct source to compile.
     * @param packager an object implementing the Packager interface.  This object packages and (usually) persists the target form.
     * @param adjunctModuleName the name of the module in which to compile the adjunct.
     * @return the highest error condition experienced
     */
    CompilerMessage.Severity compileAdjunct(AdjunctSource adjunctSource, Packager packager, ModuleName adjunctModuleName) {

        if (DEBUG_CONCURRENT_COMPILATION) {
            // increment the total number of adjuncts compiled, and the number
            // of active threads compiling adjuncts
            synchronized (classMutex) {
                adjunctCompileCount++;
                System.err.println("adjunct compile count: " + adjunctCompileCount);
                numThreadsCompiling++;
                System.err.println("Num threads: " + numThreadsCompiling);
            }

            try {
                // perform the actual compilation
                return compileAdjunctHelper(adjunctSource, packager, adjunctModuleName);

            } finally {
                // this thread has finished compiling, so decrement the number of
                // active threads compiling adjuncts
                synchronized (classMutex) {
                    numThreadsCompiling--;
                }
            }
        }
        else {
            return compileAdjunctHelper(adjunctSource, packager, adjunctModuleName);
        }
    }

    /**
     * Compile an adjunct from an AdjunctSource.
     * @param adjunctSource the adjuct source to compile.
     * @param packager an object implementing the Packager interface.  This object packages and (usually) persists the target form.
     * @param adjunctModuleName the name of the module in which to compile the adjunct.
     * @return the highest error condition experienced
     */
    private CompilerMessage.Severity compileAdjunctHelper(AdjunctSource adjunctSource, Packager packager, ModuleName adjunctModuleName) {

        if (adjunctModuleName == null) {
            throw new NullPointerException("CALCompiler.compile: Module name must not be null.");
        }

        if (packager == null) {
            throw new IllegalArgumentException("CALCompiler.compile: must have a packager.");
        }
        //Store current packager
        this.packager = packager;

        if (adjunctSource instanceof AdjunctSource.FromText) {
            setInputStream(((AdjunctSource.FromText)adjunctSource).getReader(), null);
        }

        CompilerMessageLogger oldLogger = msgLogger;
        CompilerMessageLogger compileLogger = new MessageLogger(true);  // Internal compiler logger
        setCompilerMessageLogger(compileLogger);

        // Compile and catch any fatal errors
        //todoBI we need a method of adding scs to a module, and then removing them, without
        //typechecking the whole module. In other words, adding adjuncts as an undoable operation.
        try {
            try {

                try {
                    // Initialize packaging of the adjunct.  This will switch to the
                    // adjunct module and do necessary housekeeping.
                    packager.initAdjunct(adjunctModuleName);
                } catch (Packager.PackagerException e) {
                    logMessage(new CompilerMessage(new SourceRange(adjunctModuleName), new MessageKind.Fatal.ModuleNotInWorkspace(adjunctModuleName)));
                }

                ParseTreeNode parseTree = null;
                if (adjunctSource instanceof AdjunctSource.FromText) {

                    // Call the parser
                    parser.adjunct();
                    parseTree = (ParseTreeNode)parser.getAST();

                } else if (adjunctSource instanceof AdjunctSource.FromSourceModel) {

                    parseTree = ((AdjunctSource.FromSourceModel)adjunctSource).toParseTreeNode();

                } else {
                    throw new IllegalArgumentException(
                            "CALCompiler.compileAdjunct - cannot handle adjunct source of type " + adjunctSource.getClass());
                }

                //only invoke the tree parser if there are no errors when parsing the module. Otherwise the
                //tree parser is sure to fail (in a fatal error) and so there is no point doing this.
                if (msgLogger.getNErrors() == 0) {
                    //Walk the parse tree as a sanity check on the generated AST and of the tree parser
                    treeParser.adjunct(parseTree);
                }

                //We must type check the adjunct in order to resolve unqualified references,
                //and lift lambdas and local functions defined within the body of the adjunct.
                //We type check an outer defn list in order to have a place to attach lifted lambdas.
                typeChecker.checkAdjunct(parseTree, adjunctModuleName);

                ExpressionGenerator expressionGenerator = new ExpressionGenerator(this, packager.getModuleTypeInfo(adjunctModuleName), true);
                expressionGenerator.generateCode(parseTree);

                //
                // Notify the packager that the adjunct compilation is done.
                //
                packager.wrapAdjunct(msgLogger);     // throws PackagerException..

            } catch (antlr.RecognitionException e) {
                // Recognition (syntax) error
                final SourceRange sourceRange = CALParser.makeSourceRangeFromException(e);
                logMessage(new CompilerMessage(sourceRange, new MessageKind.Error.SyntaxError(), e));

            } catch (antlr.TokenStreamException e) {
                // Bad token stream.  Maybe a bad token (eg. a stray "$" sign)
                logMessage(new CompilerMessage(new SourceRange(adjunctModuleName), new MessageKind.Error.BadTokenStream(), e));
            }

        } catch (AbortCompilation e) {

            //compilation aborted

        } catch (Exception e) {
            try {

                if (msgLogger.getNErrors() > 0 || e instanceof UnableToResolveForeignEntityException) {

                    // If the exception is an UnableToResolveForeignEntityException, there is
                    // a CompilerMessage inside that we should be logging.
                    if (e instanceof UnableToResolveForeignEntityException) {
                        try {
                            logMessage(((UnableToResolveForeignEntityException)e).getCompilerMessage());
                        } catch (AbortCompilation ace) {
                            //logMessage can throw a AbortCompilation if a FATAL message was sent.
                        }
                    }

                    //if an error occurred previously, we continue to compile the program to try to report additional
                    //meaningful compilation errors. However, this can produce spurious exceptions related to the fact
                    //that the program state does not satisfy preconditions because of the initial error(s). We don't
                    //report the spurious exception as an internal coding error.
                    logMessage(new CompilerMessage(new SourceRange(adjunctModuleName), new MessageKind.Info.UnableToRecover()));
                } else {
                    // Major failure - internal coding error
                    logMessage(new CompilerMessage(new MessageKind.Fatal.InternalCodingError(), e));
                }
            } catch (AbortCompilation ace) {
                // Yeah, yeah, we know
                //raiseError will throw a AbortCompilation since a FATAL message was sent.
            }
        } finally {
            // replace the old logger.
            setCompilerMessageLogger(oldLogger);

            try {
                oldLogger.logMessages(compileLogger);
            } catch (AbortCompilation e) {
                // too many errors for the ol' logger.
            }
        }

        //close the package if there have been no errors otherwise abort
        try {
            if (compileLogger.getNErrors() == 0) {
                packager.close(msgLogger);
            } else {
                packager.abort(msgLogger);
            }

        } catch (org.openquark.cal.compiler.Packager.PackagerException e) {
            try {
                logMessage(new CompilerMessage(new SourceRange(adjunctModuleName), new MessageKind.Error.UnableToCloseModuleAndPackage(e.toString())));
            } catch (AbortCompilation ace) {
            }
        }

        return compileLogger.getMaxSeverity();
    }

    /**
     * Parse the code text and return the list of identifiers that occur within it.
     * This method is used for the initial syntax checking and qualification of
     * text within a code gem.
     *
     * The identifiers returned include free symbols i.e. the symbols in the expression that
     * are not defined within the expression itself (e.g. by a let definition). The free symbols can
     * be either qualified (Prelude.x, Prelude.Boolean) or unqualified (True, Eq, sin). Those that are
     * variables which do not successfully resolve to a top-level symbol must be arguments of the code
     * expression.
     *
     * The list is ordered by increasing source positions of identifiers.
     *
     * Example:
     *     " let x :: Boolean; x = and Prelude.True False;
     *       in and x y ;"
     *     Returns identifiers : Boolean, and, Prelude.True, False, and, y
     *
     * Note: Any compiler messages held by this compiler will be lost
     *
     * @param codeGemBodyText the text of the body of the code gem i.e. what the user actually typed
     * @param moduleName the name of the module in which the code gem is considered to exist.
     * @param moduleNameResolver the module name resolver for the module in which the code gem is considered to exist.
     * @return List (SourceIdentifier) the name, type and position of identifiers encountered in
     *  the expression. Returns null if the expression does not parse.
     */
    List<SourceIdentifier> findIdentifiersInExpression(String codeGemBodyText, ModuleName moduleName, ModuleNameResolver moduleNameResolver) {
        if ((codeGemBodyText == null) || (moduleName == null)) {
            throw new NullPointerException();
        }

        java.io.StringReader sourceReader = new java.io.StringReader(codeGemBodyText);
        setInputStream(sourceReader, null);

        SourceIdentifierFinder<SourceIdentifier, SourceIdentifier> identifierFinder = new SourceIdentifierFinder.AllIdentifierFinder();
        return identifierFinder.findIdentifiersInUnparsedExpression(parser, treeParser, msgLogger, moduleName, moduleNameResolver);
    }

    /**
     * Parse the specified module and return all identifier references which need to be affected
     * in order to rename the specified top-level identifier without conflicts.
     *
     * The list of objects returned (RenameData) indicates the changes which must occur in order
     * to properly rename the specified symbol (ie: the results will include renamings for references
     * to the requested symbol, and may include renamings for local variables if any conflict with the
     * new name).
     *
     * Example:
     *   Renaming "s" to "r" in the expression
     *
     *   "let r  = 1.0;
     *        r2 = if (s r) then 2.0 else 0.0;
     *    in
     *       case x of
     *           (r, r3) -> and (s r) (r2 < r3);
     *       ;"
     *
     *   Will return renamings to "r" for references to "s",
     *               renamings to "r4" for case variable "r" ("r2" is bound in let, "r3" bound in case)
     *               renamings to "r3" for let variable "r" ("r2" is bound in let)
     *
     * Notes: The list returned is ordered by increasing source position of identifier names.
     *        Any compiler messages held by this compiler will be lost
     *
     * @param source module source to parse
     * @param oldName old name of the identifier
     * @param newName new name of the identifier
     * @param category category of the identifier
     * @return List the name and position of renamings encountered in
     *  the expression. Returns null if the expression does not parse.
     */
    List<SourceModification> findRenamingsInModule(ModuleSourceDefinition source, QualifiedName oldName, QualifiedName newName, SourceIdentifier.Category category) {

        Reader moduleReader = source.getSourceReader(new Status("Read module status."));
        if (moduleReader == null) {
            return Collections.emptyList();
        }
        moduleReader = new BufferedReader(moduleReader);

        try {
            setInputStream(moduleReader, source.getModuleName());

            SourceIdentifierFinder<SourceModification, String> identifierFinder = new RenamedIdentifierFinder(oldName, newName, category);
            return identifierFinder.findIdentifiersInUnparsedModule(parser, treeParser);

        } finally {
            try {
                // Close the input stream
                moduleReader.close();
            } catch (IOException e) {
                logMessage(new CompilerMessage(new SourceRange(source.getModuleName()), new MessageKind.Error.UnableToCloseModule(source.getModuleName())));
            }
        }
    }

    /**
     * Parse the specified expression and return all identifier references which need to be affected in order
     * to rename the specified identifier without conflicts.
     *
     * The list of objects returned (RenameData) indicates the changes which must occur in order
     * to properly rename the specified symbol (ie: the results will include renamings for references
     * to the requested symbol, and may include renamings for local variables if any conflict with the
     * new name).
     *
     * Example:
     *   Renaming "s" to "r" in the expression
     *
     *   "let r  = 1.0;
     *        r2 = if (s r) then 2.0 else 0.0;
     *    in
     *       case x of
     *           (r, r3) -> and (s r) (r2 < r3);
     *       ;"
     *
     *   Will return renamings to "r" for references to "s",
     *               renamings to "r4" for case variable "r" ("r2" is bound in let, "r3" bound in case)
     *               renamings to "r3" for let variable "r" ("r2" is bound in let)
     *
     * @param expressionDefinition the expression to parse
     * @param moduleName The name of the module that this expression belongs to
     * @param moduleNameResolver the module name resolver for the module to which this expression belongs.
     * @param qualificationMap The qualification map for this code expression if it has one, or null if it does not.
     * @param oldName old name of the identifier
     * @param newName new name of the identifier
     * @param category category of the identifier
     * @return List the name, type and position of identifiers encountered in  the expression,
     * ordered by increasing source position of identifier names. Returns null if the expression does not parse.
     */
    List<SourceModification> findRenamingsInExpression(String expressionDefinition, ModuleName moduleName, ModuleNameResolver moduleNameResolver, CodeQualificationMap qualificationMap, QualifiedName oldName, QualifiedName newName, SourceIdentifier.Category category) {
        Reader expressionReader = new StringReader(expressionDefinition);
        if (expressionReader == null) {
            return Collections.emptyList();
        }
        expressionReader = new BufferedReader(expressionReader);
        setInputStream(expressionReader, null);

        SourceIdentifierFinder<SourceModification, String> identifierFinder = new RenamedIdentifierFinder(oldName, newName, category, qualificationMap);
        return identifierFinder.findIdentifiersInUnparsedExpression(parser, treeParser, msgLogger, moduleName, moduleNameResolver);
    }

    /**
     * Return the packager in use.
     * Creation date: (3/16/00 9:43:14 PM)
     * @return Packager the current packager
     */
    Packager getPackager() {
        return packager;
    }

    /**
     * Set the packager associated with the compiler.
     * @param packager
     */
    void setPackager(Packager packager) {
        this.packager = packager;
    }

    /**
     * Returns the type checker that was used during compilation. This type checker will have
     * information related to the types found in the compilation of the CAL module.
     * Creation date: (10/13/00 10:01:31 AM)
     * @return CALTypeChecker
     */
    CALTypeChecker getTypeChecker() {
        return typeChecker;
    }

    /**
     * @return an instance of the deprecation scanner which performs a pass for finding deprecated entities.
     */
    DeprecationScanner getDeprecationScanner() {
        return deprecationScanner;
    }

    /**
     * Log the given compiler message.
     * If the message has severity FATAL, then an exception is thrown to immediately halt
     * compilation.  Otherwise, if the message has severity ERROR and the maximum number of
     * such errors has been reported then a 'too many errors' exception is thrown.
     * @param defaultModuleNameForSourceRange the name of the module with respect to which the message will be reported, provided that
     * 1) the compiler message does not have a source range set, and 2) the current module is not set.
     * If the current module is set, it will be used for the source range
     * @param compilerMessage the message to log.
     */
    private void logMessage(ModuleName defaultModuleNameForSourceRange, CompilerMessage compilerMessage) {
        if (compilerMessage.getSourceRange() == null){
            //handle the case of null source ranges by attempting to put a source range that at least identifies
            //the module in which the compiler message was logged, even if its precise position cannot be determined.
            final CALTypeChecker typeChecker = getTypeChecker();

            ModuleName newSourceRangeModuleName = null;

            // Attempt to get the current module name from the type checker.
            if (typeChecker != null) {
                newSourceRangeModuleName = typeChecker.getCurrentModuleName();  // can be null
            }

            // If unassigned, assign from the module name which was passed in.
            if (newSourceRangeModuleName == null) {
                newSourceRangeModuleName = defaultModuleNameForSourceRange;  // can be null.
            }

            // If a non-null source range module name was assigned above, use it.
            if (newSourceRangeModuleName != null) {
                compilerMessage = compilerMessage.copy(new SourceRange(newSourceRangeModuleName));
            }
        }
        msgLogger.logMessage (compilerMessage);
    }

    /**
     * Log the given compiler message.
     * If the message has severity FATAL, then an exception is thrown to immediately halt
     * compilation.  Otherwise, if the message has severity ERROR and the maximum number of
     * such errors has been reported then a 'too many errors' exception is thrown.
     * @param compilerMessage CompilerMessage the error/warning/info message
     * @throws CompilerMessage.AbortCompilation if a FATAL message is received or
     *      too many ERROR messages are received
     */
    void logMessage(CompilerMessage compilerMessage) {
        logMessage(null, compilerMessage);
    }

    /**
     * Set a new input stream for the compiler.
     * @param inStream java.io.Reader the input stream
     * @param compileLocation the location of the source of the code currently being compiled, or null if none.
     */
    private void setInputStream(Reader inStream, ModuleName compileLocation) {

        // Now set up a new multiplexed lexer and get it hooked up to the compiler
        lexer = new CALMultiplexedLexer(this, inStream, null);

        // Create a new queue for tokens presented to the parser (from the multiplexed lexer)
        parser.setTokenBuffer(new antlr.TokenBuffer(lexer));

        if (compileLocation != null) {
            setCompileSourceName(compileLocation.toSourceText());
        } else {
            setCompileSourceName(null);
        }
    }

    /**
     * Set the name of the source of the code currently being compiled.
     * @param compileSourceName The name of the current compile source, or null if none.
     */
    private void setCompileSourceName(String compileSourceName) {
        lexer.setFilename(compileSourceName);
        parser.setFilename(compileSourceName);
    }

    /**
     * Set the message logger.
     * @param logger the logger to set, or null to use a default logger.
     */
    void setCompilerMessageLogger (CompilerMessageLogger logger) {
        if (logger == null) {
            logger = new MessageLogger();
        }
        this.msgLogger = logger;
    }

    CompilerMessageLogger getMessageLogger () {
        return msgLogger;
    }

    /**
     * Get the module dependency graph for a given set of modules.
     *
     * @param parseInfo the parse info for the module source definitions.
     * @param packagedModuleNamesSet set of module names which already exist in the packager
     * @param preparseBrokenModuleNameSet the names of modules with errors during dependency ordering analysis.
     *   This collection will be populated by this method.
     * @return Graph the module dependency graph.
     *   Note that this graph may have components where the number of modules in the component > 1.
     *   This indicates a recursive module dependency among those modules.
     */
    private Graph<ModuleName> getModuleDependencyGraph(
            SourceDefinitionsHeaderParseInfo parseInfo,
            Set<ModuleName> packagedModuleNamesSet,
            Set<ModuleName> preparseBrokenModuleNameSet) {

        // Perform dependency analysis to divide up the modules defined in this compilation unit into
        // a topologically ordered set of strongly connected components.

        VertexBuilderList<ModuleName> vertexBuilderList =
            makeModuleDependencyGraphBuilderList(parseInfo, packagedModuleNamesSet, preparseBrokenModuleNameSet);

        // should never fail. It is a redundant check since makeModuleDependencyGraphBuilderList should throw an exception otherwise.
        if (!vertexBuilderList.makesValidGraph()) {
            throw new IllegalStateException("Internal coding error during dependency analysis.");
        }

        Graph<ModuleName> g = new Graph<ModuleName>(vertexBuilderList);

        g = g.calculateStronglyConnectedComponents();

        return g;
    }

    /**
     * Get the order in which the given modules should be compiled such that any dependee modules come before their dependents.
     * @param moduleDependencyGraph the module dependency graph.
     * @param parseInfo the parse info for the module source definitions.
     *   Used for source positions while error checking.
     * @param preparseBrokenModuleNameSet (Set of String) the names of modules with errors during dependency ordering analysis.
     *   This collection will be populated by this method.
     * @return the module names in the order in which they should be compiled.
     */
    private ModuleName[] getModuleDependencyOrder(
            Graph<ModuleName> moduleDependencyGraph,
            SourceDefinitionsHeaderParseInfo parseInfo,
            Set<ModuleName> preparseBrokenModuleNameSet) {

        // Construct the module names array.
        int nComponents = moduleDependencyGraph.getNStronglyConnectedComponents();
        List<ModuleName> moduleNameList = new ArrayList<ModuleName>(nComponents);

        for (int i = 0; i < nComponents; ++i) {
            Graph<ModuleName>.Component component = moduleDependencyGraph.getStronglyConnectedComponent(i);

            if (component.size() != 1) {

                StringBuilder cyclicNames = new StringBuilder();
                ModuleName cyclicModuleName = null;

                for (int j = 0, componentSize = component.size(); j < componentSize; ++j) {

                    if (j > 0) {
                        cyclicNames.append(", ");
                    }

                    cyclicModuleName = component.getVertex(j).getName();
                    cyclicNames.append(cyclicModuleName);
                    moduleNameList.add(cyclicModuleName);
                    preparseBrokenModuleNameSet.add(cyclicModuleName);
                }

                ParseTreeNode cyclicModuleNameNode = parseInfo.getModuleNameToHeaderDefnNodeMap().get(cyclicModuleName);
                MessageKind errorMessage = new MessageKind.Error.CyclicDependenciesBetweenModules(cyclicNames.toString());
                if (cyclicModuleNameNode != null) {
                    logMessage(new CompilerMessage(cyclicModuleNameNode.getChild(0), errorMessage));
                } else {
                    logMessage(new CompilerMessage(new SourceRange(cyclicModuleName), errorMessage));
                }

            } else {
                Vertex<ModuleName> vertex = component.getVertex(0);
                ModuleName currentModuleName = vertex.getName();
                moduleNameList.add(currentModuleName);
            }

        }

        return moduleNameList.toArray(new ModuleName[moduleNameList.size()]);
    }

    /**
     * Makes the vertex builder list for modules in this compilation unit.
     *
     * @param parseInfo the parse info for the module source definitions.
     * @param packagedModuleNamesSet set (String) of module names which already exist in the packager
     * @param preparseBrokenModuleNameSet (Set of String) the names of modules with errors during dependency ordering analysis.
     *   This collection will be populated by this method.
     * @return VertexBuilderList
     */
    private VertexBuilderList<ModuleName> makeModuleDependencyGraphBuilderList(
            SourceDefinitionsHeaderParseInfo parseInfo,
            Set<ModuleName> packagedModuleNamesSet,
            Set<ModuleName> preparseBrokenModuleNameSet) {

        Map<ModuleName, ParseTreeNode> moduleNameToHeaderDefnNodeMap = parseInfo.getModuleNameToHeaderDefnNodeMap();
        SortedMap<ModuleName, ImportNodeInfo[]> moduleNameToImportNodeInfoMap = parseInfo.getModuleNameToImportNodeInfoMap();
        Set<ModuleName> nonParseableNamesSet = parseInfo.getNonParseableModuleNamesSet();

        VertexBuilderList<ModuleName> vertexBuilderList = new VertexBuilderList<ModuleName>();

        // Build up the names of the modules in the set
        Set<ModuleName> moduleNamesSet = moduleNameToImportNodeInfoMap.keySet();

        // Go through each module and check that the imports make sense.  If so, add it to the graph.
        for (final Map.Entry<ModuleName, ImportNodeInfo[]> entry : moduleNameToImportNodeInfoMap.entrySet()) {

            final ModuleName moduleName = entry.getKey();

            // Skip any modules determined to be "broken"
            // For example, these may be modules which parse but whose module source names don't correspond to their definitions.
            if (preparseBrokenModuleNameSet.contains(moduleName)) {
                continue;
            }

            ImportNodeInfo[] importNodeInfoArray = entry.getValue();
            Set<ModuleName> importedModuleNamesSet = new HashSet<ModuleName>();

            boolean importsPrelude = false;
            boolean broken = false;
            for (final ImportNodeInfo importNodeInfo : importNodeInfoArray) {
                ModuleName importedModuleName = importNodeInfo.getImportName();

                if (moduleName.equals(importedModuleName)) {
                    SourceRange sourceRange = getSourceRangeForCompilerMessage(importNodeInfo.getImportedModuleNameNode(), moduleName);
                    logMessage(new CompilerMessage(sourceRange, new MessageKind.Error.AttemptToImportModuleIntoItself(importedModuleName)));
                    broken = true;
                }

                if (importedModuleName.equals(CAL_Prelude.MODULE_NAME)) {
                    importsPrelude = true;
                }

                if (!moduleNamesSet.contains(importedModuleName) &&
                    !nonParseableNamesSet.contains(importedModuleName) &&
                    !packagedModuleNamesSet.contains(importedModuleName)) {

                    final Set<ModuleName> potentialModules = new HashSet<ModuleName>();
                    potentialModules.addAll(moduleNamesSet);
                    potentialModules.addAll(nonParseableNamesSet);
                    potentialModules.addAll(packagedModuleNamesSet);
                    final ModuleNameResolver resolverForPotentialModules = ModuleNameResolver.make(potentialModules);

                    ModuleNameResolver.ResolutionResult resolutionResult = resolverForPotentialModules.resolve(importedModuleName);

                    SourceRange sourceRange = getSourceRangeForCompilerMessage(importNodeInfo.getImportedModuleNameNode(), moduleName);

                    if (resolutionResult.isKnownUnambiguous()) {
                        logMessage(new CompilerMessage(sourceRange, new MessageKind.Error.UnresolvedExternalModuleImportWithOneSuggestion(importedModuleName, resolutionResult.getResolvedModuleName())));
                    } else if (resolutionResult.isAmbiguous()) {
                        logMessage(new CompilerMessage(sourceRange, new MessageKind.Error.UnresolvedExternalModuleImportWithMultipleSuggestions(importedModuleName, resolutionResult.getPotentialMatches())));
                    } else {
                        logMessage(new CompilerMessage(sourceRange, new MessageKind.Error.UnresolvedExternalModuleImportWithNoSuggestions(importedModuleName)));
                    }

                    broken = true;
                    // do not add this to dependency graph, since it doesn't exist as a vertex
                    continue;
                }

                if (!importedModuleNamesSet.add(importedModuleName)) {
                    SourceRange sourceRange = getSourceRangeForCompilerMessage(importNodeInfo.getImportedModuleNameNode(), moduleName);
                    logMessage(new CompilerMessage(sourceRange, new MessageKind.Error.RepeatedImportOfModule(importedModuleName)));
                    broken = true;
                }

                // Now check whether the module being imported is deprecated
                SourceRange sourceRange = getSourceRangeForCompilerMessage(importNodeInfo.getImportedModuleNameNode(), moduleName);
                checkResolvedModuleReference(importedModuleName, sourceRange);
            }

            if (!moduleName.equals(CAL_Prelude.MODULE_NAME) && !importsPrelude) {
                //non-Prelude modules must import the Prelude.
                ParseTreeNode moduleDefnNode = moduleNameToHeaderDefnNodeMap.get(moduleName);
                logMessage(new CompilerMessage(moduleDefnNode, new MessageKind.Error.ModuleMustImportOtherModule(moduleName, CAL_Prelude.MODULE_NAME)));
                broken = true;
            }

            if (broken) {
                preparseBrokenModuleNameSet.add(moduleName);
            }

            vertexBuilderList.add(new VertexBuilder<ModuleName>(moduleName, importedModuleNamesSet));
        }

        // Add the names of all packaged and non parseable modules to the graph for completion

        for (final ModuleName moduleName : nonParseableNamesSet) {

            vertexBuilderList.add(new VertexBuilder<ModuleName>(moduleName, new HashSet<ModuleName>()));
        }
        for (final ModuleName moduleName : packagedModuleNamesSet) {

            vertexBuilderList.add(new VertexBuilder<ModuleName>(moduleName, new HashSet<ModuleName>()));
        }

        return vertexBuilderList;
    }

    /**
     * @param parseTreeNode the parseTreeNode associated with a CompilerMessage.  May be null.
     * @param defaultModuleNameForSourceRange the module associated with the compiler message.
     * @return a SourceRange to use with the CompilerMessage.
     * If there is SourceRange available from the provided parseTreeNode (if any), it will be returned.
     * Otherwise a default SourceRange will be constructed from the currently compiling module name.
     */
    static SourceRange getSourceRangeForCompilerMessage(ParseTreeNode parseTreeNode, ModuleName defaultModuleNameForSourceRange) {

        if (parseTreeNode != null) {
            SourceRange assemblySourceRange = parseTreeNode.getAssemblySourceRange();
            if (assemblySourceRange != null) {
                return assemblySourceRange;
            }
        }

        return new SourceRange(defaultModuleNameForSourceRange);
    }

    /**
     * Performs late static checks on a module reference that has already been resolved.
     *
     * Currently, this performs a deprecation check on a module reference, logging a warning message if the module is deprecated.
     * @param moduleName the module name to check.
     * @param sourceRange the source range of the reference.
     */
    void checkResolvedModuleReference(final ModuleName moduleName, final SourceRange sourceRange) {
        if (getDeprecationScanner().isModuleDeprecated(moduleName)) {
            logMessage(new CompilerMessage(sourceRange, new MessageKind.Warning.DeprecatedModule(moduleName)));
        }
    }

    /**
     * Return the time stamp of the currently compiling module.
     * @return long
     */
    long getCurrentModuleTimeStamp () {
        return currentModuleTimeStamp;
    }
    /**
     * The compiler will use the CAL based optimizer.
     */
    void useOptimizer() {
        packager.useOptimizer();
    }
}