text:
//===--- ImportResolution.cpp - Import Resolution -------------------------===//
//
// This source file is part of the Swift.org open source project
//
// Copyright (c) 2014 - 2020 Apple Inc. and the Swift project authors
// Licensed under Apache License v2.0 with Runtime Library Exception
//
// See https://swift.org/LICENSE.txt for license information
// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
//
//===----------------------------------------------------------------------===//
//
//  This file performs import resolution.
//
//===----------------------------------------------------------------------===//

#define DEBUG_TYPE "swift-import-resolution"
#include "swift/AST/ASTWalker.h"
#include "swift/AST/DiagnosticsSema.h"
#include "swift/AST/ModuleLoader.h"
#include "swift/AST/ModuleNameLookup.h"
#include "swift/AST/NameLookup.h"
#include "swift/AST/SourceFile.h"
#include "swift/AST/SubstitutionMap.h"
#include "swift/AST/TypeCheckRequests.h"
#include "swift/Basic/Statistic.h"
#include "swift/ClangImporter/ClangModule.h"
#include "swift/Parse/Parser.h"
#include "swift/Subsystems.h"
#include "clang/Basic/Module.h"
#include "llvm/ADT/DenseMap.h"
#include "llvm/ADT/TinyPtrVector.h"
#include "llvm/ADT/Twine.h"
#include "llvm/Support/Debug.h"
#include "llvm/Support/Host.h"
#include "llvm/Support/Path.h"
#include "llvm/Support/SaveAndRestore.h"
#include <algorithm>
#include <system_error>
using namespace swift;

//===----------------------------------------------------------------------===//
// MARK: ImportResolver and supporting types
//===----------------------------------------------------------------------===//

using ImportedModule = ModuleDecl::ImportedModule;
using ImportedModuleDesc = SourceFile::ImportedModuleDesc;
using ImportOptions = SourceFile::ImportOptions;
using ImportFlags = SourceFile::ImportFlags;

namespace {
/// Represents an import which the ImportResolver knows exists, but which has
/// not yet had its options checked, module loaded, or cross-imports found.
///
/// An UnboundImport may represent a physical ImportDecl written in the
/// source, or it may represent a cross-import overlay that has been found and
/// needs to be loaded.
struct UnboundImport {
  /// The source location to use when diagnosing errors for this import.
  SourceLoc importLoc;

  /// The options for this import, such as "exported" or
  /// "implementation-only". Use this field, not \c attrs, to determine the
  /// behavior expected for this import.
  ImportOptions options;

  /// If \c options includes \c PrivateImport, the filename we should import
  /// private declarations from.
  StringRef privateImportFileName;

  /// The module names being imported. There will usually be just one for the
  /// top-level module, but a submodule import will have more.
  ModuleDecl::AccessPathTy modulePath;

  /// If this is a scoped import, the names of the declaration being imported;
  /// otherwise empty. (Currently the compiler doesn't support nested scoped
  /// imports, so there should always be zero or one elements, but
  /// \c AccessPathTy is the common currency type for this.)
  ModuleDecl::AccessPathTy declPath;

  // Names of explicitly imported SPI groups via @_spi.
  ArrayRef<Identifier> spiGroups;

  /// If this UnboundImport directly represents an ImportDecl, contains the
  /// ImportDecl it represents. This should only be used for diagnostics and
  /// for updating the AST; if you want to read information about the import,
  /// get it from the other fields in \c UnboundImport rather than from the
  /// \c ImportDecl.
  ///
  /// If this UnboundImport represents a cross-import, contains the declaring
  /// module's \c ModuleDecl.
  PointerUnion<ImportDecl *, ModuleDecl *> importOrUnderlyingModuleDecl;

  NullablePtr<ImportDecl> getImportDecl() const {
    return importOrUnderlyingModuleDecl.is<ImportDecl *>() ?
           importOrUnderlyingModuleDecl.get<ImportDecl *>() : nullptr;
  }

  NullablePtr<ModuleDecl> getUnderlyingModule() const {
    return importOrUnderlyingModuleDecl.is<ModuleDecl *>() ?
           importOrUnderlyingModuleDecl.get<ModuleDecl *>() : nullptr;
  }

  /// Create an UnboundImport for a user-written import declaration.
  explicit UnboundImport(ImportDecl *ID);

  /// Create an UnboundImport for a cross-import overlay.
  explicit UnboundImport(ASTContext &ctx,
                         const UnboundImport &base, Identifier overlayName,
                         const ImportedModuleDesc &declaringImport,
                         const ImportedModuleDesc &bystandingImport);

  /// Diagnoses if the import would simply load the module \p SF already
  /// belongs to, with no actual effect.
  ///
  /// Some apparent self-imports do actually load a different module; this
  /// method allows them.
  bool checkNotTautological(const SourceFile &SF);

  /// Make sure the module actually loaded, and diagnose if it didn't.
  bool checkModuleLoaded(ModuleDecl *M, SourceFile &SF);

  /// Find the top-level module for this module; that is, if \p M is the
  /// module \c Foo.Bar.Baz, this finds \c Foo.
  ///
  /// Specifically, this method returns:
  ///
  /// \li \p M if \p M is a top-level module.
  /// \li \c nullptr if \p M is a submodule of \c SF's parent module. (This
  ///     corner case can occur in mixed-source frameworks, where Swift code
  ///     can import a Clang submodule of itself.)
  /// \li The top-level parent (i.e. ancestor with no parent) module above
  ///     \p M otherwise.
  NullablePtr<ModuleDecl> getTopLevelModule(ModuleDecl *M, SourceFile &SF);

  /// Diagnose any errors concerning the \c @_exported, \c @_implementationOnly,
  /// \c \@testable, or \c @_private attributes, including a
  /// non-implementation-only import of a fragile library from a resilient one.
  void validateOptions(NullablePtr<ModuleDecl> topLevelModule, SourceFile &SF);

  /// Create an \c ImportedModuleDesc from the information in this
  /// UnboundImport.
  ImportedModuleDesc makeDesc(ModuleDecl *module) const {
    return ImportedModuleDesc({ declPath, module }, options,
                              privateImportFileName, spiGroups);
  }

private:
  void validatePrivate(ModuleDecl *topLevelModule);
  void validateImplementationOnly(ASTContext &ctx);
  void validateTestable(ModuleDecl *topLevelModule);
  void validateResilience(NullablePtr<ModuleDecl> topLevelModule,
                          SourceFile &SF);

  /// Diagnoses an inability to import \p modulePath in this situation and, if
  /// \p attrs is provided and has an \p attrKind, invalidates the attribute and
  /// offers a fix-it to remove it.
  void diagnoseInvalidAttr(DeclAttrKind attrKind, DiagnosticEngine &diags,
                           Diag<Identifier> diagID);
};

class ImportResolver final : public DeclVisitor<ImportResolver> {
  friend DeclVisitor<ImportResolver>;

  SourceFile &SF;
  ASTContext &ctx;

  /// Imports which still need their options checked, modules loaded, and
  /// cross-imports found.
  SmallVector<UnboundImport, 4> unboundImports;

  /// The list of fully bound imports.
  SmallVector<ImportedModuleDesc, 16> boundImports;

  /// All imported modules which should be considered when cross-importing.
  /// This is basically the transitive import graph, but with only top-level
  /// modules and without reexports from Objective-C modules.
  ///
  /// We use a \c SmallSetVector here because this doubles as the worklist for
  /// cross-importing, so we want to keep it in order; this is feasible
  /// because this set is usually fairly small.
  SmallSetVector<ImportedModuleDesc, 64> crossImportableModules;

  /// The subset of \c crossImportableModules which may declare cross-imports.
  ///
  /// This is a performance optimization. Since most modules do not register
  /// any cross-imports, we can usually compare against this list, which is
  /// much, much smaller than \c crossImportableModules.
  SmallVector<ImportedModuleDesc, 16> crossImportDeclaringModules;

  /// The index of the next module in \c visibleModules that should be
  /// cross-imported.
  size_t nextModuleToCrossImport = 0;

public:
  ImportResolver(SourceFile &SF) : SF(SF), ctx(SF.getASTContext()) {
    addImplicitImports();
  }

  void addImplicitImports() {
    // TODO: Support cross-module imports.
    for (auto &import : SF.getParentModule()->getImplicitImports()) {
      assert(!(SF.Kind == SourceFileKind::SIL &&
               import.Module->isStdlibModule()));
      ImportedModule importedMod{ModuleDecl::AccessPathTy(), import.Module};
      boundImports.emplace_back(importedMod, import.Options);
    }
  }

  /// Retrieve the finalized imports.
  ArrayRef<ImportedModuleDesc> getFinishedImports() const {
    return boundImports;
  }

private:
  // We only need to visit import decls.
  void visitImportDecl(ImportDecl *ID);

  // Ignore other decls.
  void visitDecl(Decl *D) {}

  template<typename ...ArgTypes>
  InFlightDiagnostic diagnose(ArgTypes &&...Args) {
    return ctx.Diags.diagnose(std::forward<ArgTypes>(Args)...);
  }

  /// Check a single unbound import, bind it, add it to \c boundImports,
  /// and add its cross-import overlays to \c unboundImports.
  void bindImport(UnboundImport &&I);

  /// Adds \p I and \p M to \c boundImports and \c visibleModules.
  void addImport(const UnboundImport &I, ModuleDecl *M);

  /// Adds \p desc and everything it re-exports to \c visibleModules using
  /// the settings from \c desc.
  void addCrossImportableModules(ImportedModuleDesc desc);

  /// * If \p I is a cross-import overlay, registers \p M as overlaying
  ///   \p I.underlyingModule in \c SF.
  /// * Discovers any cross-imports between \p I and previously bound imports,
  ///   then adds them to \c unboundImports using source locations from \p I.
  void crossImport(ModuleDecl *M, UnboundImport &I);

  /// Discovers any cross-imports between \p newImport and
  /// \p oldImports and adds them to \c unboundImports, using source
  /// locations from \p I.
  void findCrossImportsInLists(UnboundImport &I,
                               ArrayRef<ImportedModuleDesc> declaring,
                               ArrayRef<ImportedModuleDesc> bystanding,
                               bool shouldDiagnoseRedundantCrossImports);

  /// Discovers any cross-imports between \p declaringImport and
  /// \p bystandingImport and adds them to \c unboundImports, using source
  /// locations from \p I.
  void findCrossImports(UnboundImport &I,
                        const ImportedModuleDesc &declaringImport,
                        const ImportedModuleDesc &bystandingImport,
                        bool shouldDiagnoseRedundantCrossImports);

  /// Load a module referenced by an import statement.
  ///
  /// Returns null if no module can be loaded.
  ModuleDecl *getModule(ArrayRef<Located<Identifier>> ModuleID);
};
} // end anonymous namespace

//===----------------------------------------------------------------------===//
// MARK: performImportResolution
//===----------------------------------------------------------------------===//

/// performImportResolution - This walks the AST to resolve imports.
///
/// Before we can type-check a source file, we need to make declarations
/// imported from other modules available. This is done by processing top-level
/// \c ImportDecl nodes, along with related validation.
///
/// Import resolution operates on a parsed but otherwise unvalidated AST.
void swift::performImportResolution(SourceFile &SF) {
  // If we've already performed import resolution, bail.
  if (SF.ASTStage == SourceFile::ImportsResolved)
    return;

  FrontendStatsTracer tracer(SF.getASTContext().Stats,
                             "Import resolution");

  // If we're silencing parsing warnings, then also silence import warnings.
  // This is necessary for secondary files as they can be parsed and have their
  // imports resolved multiple times.
  auto &diags = SF.getASTContext().Diags;
  auto didSuppressWarnings = diags.getSuppressWarnings();
  auto shouldSuppress = SF.getParsingOptions().contains(
      SourceFile::ParsingFlags::SuppressWarnings);
  diags.setSuppressWarnings(didSuppressWarnings || shouldSuppress);
  SWIFT_DEFER { diags.setSuppressWarnings(didSuppressWarnings); };

  ImportResolver resolver(SF);

  // Resolve each import declaration.
  for (auto D : SF.getTopLevelDecls())
    resolver.visit(D);

  SF.setImports(resolver.getFinishedImports());

  SF.ASTStage = SourceFile::ImportsResolved;
  verify(SF);
}

//===----------------------------------------------------------------------===//
// MARK: Import handling generally
//===----------------------------------------------------------------------===//

void ImportResolver::visitImportDecl(ImportDecl *ID) {
  assert(unboundImports.empty());

  unboundImports.emplace_back(ID);
  while(!unboundImports.empty())
    bindImport(unboundImports.pop_back_val());
}

void ImportResolver::bindImport(UnboundImport &&I) {
  auto ID = I.getImportDecl();

  if (!I.checkNotTautological(SF)) {
    // No need to process this import further.
    if (ID)
      ID.get()->setModule(SF.getParentModule());
    return;
  }

  ModuleDecl *M = getModule(I.modulePath);
  if (!I.checkModuleLoaded(M, SF)) {
    // Can't process further. checkModuleLoaded() will have diagnosed this.
    if (ID)
      ID.get()->setModule(nullptr);
    return;
  }

  auto topLevelModule = I.getTopLevelModule(M, SF);

  I.validateOptions(topLevelModule, SF);

  if (topLevelModule && topLevelModule != M) {
    // If we have distinct submodule and top-level module, add both.
    addImport(I, M);
    addImport(I, topLevelModule.get());
  }
  else {
    // Add only the import itself.
    addImport(I, M);
  }

  crossImport(M, I);

  if (ID)
    ID.get()->setModule(M);
}

void ImportResolver::addImport(const UnboundImport &I, ModuleDecl *M) {
  auto importDesc = I.makeDesc(M);
  addCrossImportableModules(importDesc);
  boundImports.push_back(importDesc);
}

//===----------------------------------------------------------------------===//
// MARK: Import module loading
//===----------------------------------------------------------------------===//

ModuleDecl *
ImportResolver::getModule(ArrayRef<Located<Identifier>> modulePath) {
  assert(!modulePath.empty());
  auto moduleID = modulePath[0];

  // The Builtin module cannot be explicitly imported unless we're a .sil file.
  if (SF.Kind == SourceFileKind::SIL &&
      moduleID.Item == ctx.TheBuiltinModule->getName())
    return ctx.TheBuiltinModule;

  // If the imported module name is the same as the current module,
  // skip the Swift module loader and use the Clang module loader instead.
  // This allows a Swift module to extend a Clang module of the same name.
  //
  // FIXME: We'd like to only use this in SIL mode, but unfortunately we use it
  // for clang overlays as well.
  if (moduleID.Item == SF.getParentModule()->getName() &&
      modulePath.size() == 1) {
    if (auto importer = ctx.getClangModuleLoader())
      return importer->loadModule(moduleID.Loc, modulePath);
    return nullptr;
  }

  return ctx.getModule(modulePath);
}

NullablePtr<ModuleDecl>
UnboundImport::getTopLevelModule(ModuleDecl *M, SourceFile &SF) {
  if (modulePath.size() == 1)
    return M;

  // If we imported a submodule, import the top-level module as well.
  Identifier topLevelName = modulePath.front().Item;
  ModuleDecl *topLevelModule = SF.getASTContext().getLoadedModule(topLevelName);

  if (!topLevelModule) {
    // Clang can sometimes import top-level modules as if they were
    // submodules.
    assert(!M->getFiles().empty() &&
           isa<ClangModuleUnit>(M->getFiles().front()));
    return M;
  }

  if (topLevelModule == SF.getParentModule())
    // This can happen when compiling a mixed-source framework (or overlay)
    // that imports a submodule of its C part.
    return nullptr;

  return topLevelModule;
}

//===----------------------------------------------------------------------===//
// MARK: Implicit imports
//===----------------------------------------------------------------------===//

ArrayRef<ImplicitImport>
ModuleImplicitImportsRequest::evaluate(Evaluator &evaluator,
                                       ModuleDecl *module) const {
  SmallVector<ImplicitImport, 4> imports;

  auto &ctx = module->getASTContext();
  auto &importInfo = module->getImplicitImportInfo();

  // Add an implicit stdlib if needed.
  switch (importInfo.StdlibKind) {
  case ImplicitStdlibKind::None:
    break;
  case ImplicitStdlibKind::Builtin:
    imports.emplace_back(ctx.TheBuiltinModule);
    break;
  case ImplicitStdlibKind::Stdlib: {
    auto *stdlib = ctx.getStdlibModule(/*loadIfAbsent*/ true);
    assert(stdlib && "Missing stdlib?");
    imports.emplace_back(stdlib);
    break;
  }
  }

  // Add any modules we were asked to implicitly import.
  for (auto moduleName : importInfo.ModuleNames) {
    auto *importModule = ctx.getModule({{moduleName, SourceLoc()}});
    if (!importModule) {
      ctx.Diags.diagnose(SourceLoc(), diag::sema_no_import, moduleName.str());
      if (ctx.SearchPathOpts.SDKPath.empty() &&
          llvm::Triple(llvm::sys::getProcessTriple()).isMacOSX()) {
        ctx.Diags.diagnose(SourceLoc(), diag::sema_no_import_no_sdk);
        ctx.Diags.diagnose(SourceLoc(), diag::sema_no_import_no_sdk_xcrun);
      }
      continue;
    }
    imports.emplace_back(importModule);
  }

  // Add any pre-loaded modules.
  for (auto &module : importInfo.AdditionalModules) {
    imports.emplace_back(module.first, module.second ? ImportFlags::Exported
                                                     : ImportOptions());
  }

  auto *clangImporter =
      static_cast<ClangImporter *>(ctx.getClangModuleLoader());

  // Implicitly import the bridging header module if needed.
  auto bridgingHeaderPath = importInfo.BridgingHeaderPath;
  if (!bridgingHeaderPath.empty() &&
      !clangImporter->importBridgingHeader(bridgingHeaderPath, module)) {
    auto *headerModule = clangImporter->getImportedHeaderModule();
    assert(headerModule && "Didn't load bridging header?");
    imports.emplace_back(headerModule, ImportFlags::Exported);
  }

  // Implicitly import the underlying Clang half of this module if needed.
  if (importInfo.ShouldImportUnderlyingModule) {
    auto *underlyingMod = clangImporter->loadModule(
        SourceLoc(), {Located<Identifier>(module->getName(), SourceLoc())});
    if (underlyingMod) {
      imports.emplace_back(underlyingMod, ImportFlags::Exported);
    } else {
      ctx.Diags.diagnose(SourceLoc(), diag::error_underlying_module_not_found,
                         module->getName());
    }
  }

  return ctx.AllocateCopy(imports);
}

//===----------------------------------------------------------------------===//
// MARK: Import validation (except for scoped imports)
//===----------------------------------------------------------------------===//

/// Create an UnboundImport for a user-written import declaration.
UnboundImport::UnboundImport(ImportDecl *ID)
  : importLoc(ID->getLoc()), options(), privateImportFileName(),
    modulePath(ID->getModulePath()), declPath(ID->getDeclPath()),
    importOrUnderlyingModuleDecl(ID)
{
  if (ID->isExported())
    options |= ImportFlags::Exported;

  if (ID->getAttrs().hasAttribute<TestableAttr>())
    options |= ImportFlags::Testable;

  if (ID->getAttrs().hasAttribute<ImplementationOnlyAttr>())
    options |= ImportFlags::ImplementationOnly;

  if (auto *privateImportAttr =
          ID->getAttrs().getAttribute<PrivateImportAttr>()) {
    options |= ImportFlags::PrivateImport;
    privateImportFileName = privateImportAttr->getSourceFile();
  }

  SmallVector<Identifier, 4> spiGroups;
  for (auto attr : ID->getAttrs().getAttributes<SPIAccessControlAttr>()) {
    options |= SourceFile::ImportFlags::SPIAccessControl;
    auto attrSPIs = attr->getSPIGroups();
    spiGroups.append(attrSPIs.begin(), attrSPIs.end());
  }
  this->spiGroups = ID->getASTContext().AllocateCopy(spiGroups);
}

bool UnboundImport::checkNotTautological(const SourceFile &SF) {
  // Exit early if this is not a self-import.
  if (modulePath.front().Item != SF.getParentModule()->getName() ||
      // Overlays use an @_exported self-import to load their clang module.
      options.contains(ImportFlags::Exported) ||
      // Imports of your own submodules are allowed in cross-language libraries.
      modulePath.size() != 1 ||
      // SIL files self-import to get decls from the rest of the module.
      SF.Kind == SourceFileKind::SIL)
    return true;

  ASTContext &ctx = SF.getASTContext();

  StringRef filename = llvm::sys::path::filename(SF.getFilename());
  if (filename.empty())
    ctx.Diags.diagnose(importLoc, diag::sema_import_current_module,
                       modulePath.front().Item);
  else
    ctx.Diags.diagnose(importLoc, diag::sema_import_current_module_with_file,
                       filename, modulePath.front().Item);

  return false;
}

bool UnboundImport::checkModuleLoaded(ModuleDecl *M, SourceFile &SF) {
  if (M)
    return true;

  ASTContext &ctx = SF.getASTContext();

  SmallString<64> modulePathStr;
  interleave(modulePath, [&](ImportDecl::AccessPathElement elem) {
               modulePathStr += elem.Item.str();
             },
             [&] { modulePathStr += "."; });

  auto diagKind = diag::sema_no_import;
  if (ctx.LangOpts.DebuggerSupport)
    diagKind = diag::sema_no_import_repl;
  ctx.Diags.diagnose(importLoc, diagKind, modulePathStr);

  if (ctx.SearchPathOpts.SDKPath.empty() &&
      llvm::Triple(llvm::sys::getProcessTriple()).isMacOSX()) {
    ctx.Diags.diagnose(SourceLoc(), diag::sema_no_import_no_sdk);
    ctx.Diags.diagnose(SourceLoc(), diag::sema_no_import_no_sdk_xcrun);
  }
  return false;
}

void UnboundImport::validateOptions(NullablePtr<ModuleDecl> topLevelModule,
                                    SourceFile &SF) {
  validateImplementationOnly(SF.getASTContext());

  if (auto *top = topLevelModule.getPtrOrNull()) {
    // FIXME: Having these two calls in this if condition seems dubious.
    //
    // Here's the deal: Per getTopLevelModule(), we will only skip this block
    // if you are in a mixed-source module and trying to import a submodule from
    // your clang half. But that means you're trying to @testable import or
    // @_private import part of yourself--and, moreover, a clang part of
    // yourself--which doesn't make any sense to do. Shouldn't we diagnose that?
    //
    // I'm leaving this alone for now because I'm trying to refactor without
    // changing behavior, but it smells funny.
    validateTestable(top);
    validatePrivate(top);
  }

  validateResilience(topLevelModule, SF);
}

void UnboundImport::validatePrivate(ModuleDecl *topLevelModule) {
  assert(topLevelModule);
  ASTContext &ctx = topLevelModule->getASTContext();

  if (!options.contains(ImportFlags::PrivateImport))
    return;

  if (topLevelModule->arePrivateImportsEnabled())
    return;

  diagnoseInvalidAttr(DAK_PrivateImport, ctx.Diags,
                      diag::module_not_compiled_for_private_import);
  privateImportFileName = StringRef();
}

void UnboundImport::validateImplementationOnly(ASTContext &ctx) {
  if (!options.contains(ImportFlags::ImplementationOnly) ||
      !options.contains(ImportFlags::Exported))
    return;

  // Remove one flag to maintain the invariant.
  options -= ImportFlags::ImplementationOnly;

  diagnoseInvalidAttr(DAK_ImplementationOnly, ctx.Diags,
                      diag::import_implementation_cannot_be_exported);
}

void UnboundImport::validateTestable(ModuleDecl *topLevelModule) {
  assert(topLevelModule);
  ASTContext &ctx = topLevelModule->getASTContext();

  if (!options.contains(ImportFlags::Testable) ||
      topLevelModule->isTestingEnabled() ||
      topLevelModule->isNonSwiftModule() ||
      !ctx.LangOpts.EnableTestableAttrRequiresTestableModule)
    return;

  diagnoseInvalidAttr(DAK_Testable, ctx.Diags, diag::module_not_testable);
}

void UnboundImport::validateResilience(NullablePtr<ModuleDecl> topLevelModule,
                                       SourceFile &SF) {
  if (options.contains(ImportFlags::ImplementationOnly))
    return;

  // Per getTopLevelModule(), we'll only get nullptr here for non-Swift modules,
  // so these two really mean the same thing.
  if (!topLevelModule || topLevelModule.get()->isNonSwiftModule())
    return;

  if (!SF.getParentModule()->isResilient() ||
      topLevelModule.get()->isResilient())
    return;

  ASTContext &ctx = SF.getASTContext();
  ctx.Diags.diagnose(modulePath.front().Loc,
                     diag::module_not_compiled_with_library_evolution,
                     topLevelModule.get()->getName(),
                     SF.getParentModule()->getName());
  // FIXME: Once @_implementationOnly is a real feature, we should have a fix-it
  // to add it.
}

void UnboundImport::diagnoseInvalidAttr(DeclAttrKind attrKind,
                                        DiagnosticEngine &diags,
                                        Diag<Identifier> diagID) {
  auto diag = diags.diagnose(modulePath.front().Loc, diagID,
                             modulePath.front().Item);

  auto *ID = getImportDecl().getPtrOrNull();
  if (!ID) return;
  auto *attr = ID->getAttrs().getAttribute(attrKind);
  if (!attr) return;

  diag.fixItRemove(attr->getRangeWithAt());
  attr->setInvalid();
}

evaluator::SideEffect
CheckInconsistentImplementationOnlyImportsRequest::evaluate(
    Evaluator &evaluator, ModuleDecl *mod) const {
  bool hasAnyImplementationOnlyImports =
      llvm::any_of(mod->getFiles(), [](const FileUnit *F) -> bool {
        auto *SF = dyn_cast<SourceFile>(F);
        return SF && SF->hasImplementationOnlyImports();
      });
  if (!hasAnyImplementationOnlyImports)
    return {};

  auto diagnose = [mod](const ImportDecl *normalImport,
                        const ImportDecl *implementationOnlyImport) {
    auto &diags = mod->getDiags();
    {
      InFlightDiagnostic warning =
          diags.diagnose(normalImport, diag::warn_implementation_only_conflict,
                         normalImport->getModule()->getName());
      if (normalImport->getAttrs().isEmpty()) {
        // Only try to add a fix-it if there's no other annotations on the
        // import to avoid creating things like
        // `@_implementationOnly @_exported import Foo`. The developer can
        // resolve those manually.
        warning.fixItInsert(normalImport->getStartLoc(),
                            "@_implementationOnly ");
      }
    }
    diags.diagnose(implementationOnlyImport,
                   diag::implementation_only_conflict_here);
  };

  llvm::DenseMap<ModuleDecl *, std::vector<const ImportDecl *>> normalImports;
  llvm::DenseMap<ModuleDecl *, const ImportDecl *> implementationOnlyImports;

  for (const FileUnit *file : mod->getFiles()) {
    auto *SF = dyn_cast<SourceFile>(file);
    if (!SF)
      continue;

    for (auto *topLevelDecl : SF->getTopLevelDecls()) {
      auto *nextImport = dyn_cast<ImportDecl>(topLevelDecl);
      if (!nextImport)
        continue;

      ModuleDecl *module = nextImport->getModule();
      if (!module)
        continue;

      if (nextImport->getAttrs().hasAttribute<ImplementationOnlyAttr>()) {
        // We saw an implementation-only import.
        bool isNew =
            implementationOnlyImports.insert({module, nextImport}).second;
        if (!isNew)
          continue;

        auto seenNormalImportPosition = normalImports.find(module);
        if (seenNormalImportPosition != normalImports.end()) {
          for (auto *seenNormalImport : seenNormalImportPosition->getSecond())
            diagnose(seenNormalImport, nextImport);

          // We're done with these; keep the map small if possible.
          normalImports.erase(seenNormalImportPosition);
        }
        continue;
      }

      // We saw a non-implementation-only import. Is that in conflict with what
      // we've seen?
      if (auto *seenImplementationOnlyImport =
            implementationOnlyImports.lookup(module)) {
        diagnose(nextImport, seenImplementationOnlyImport);
        continue;
      }

      // Otherwise, record it for later.
      normalImports[module].push_back(nextImport);
    }
  }
  return {};
}

//===----------------------------------------------------------------------===//
// MARK: Scoped imports
//===----------------------------------------------------------------------===//

/// Returns true if a decl with the given \p actual kind can legally be
/// imported via the given \p expected kind.
static bool isCompatibleImportKind(ImportKind expected, ImportKind actual) {
  if (expected == actual)
    return true;
  if (expected != ImportKind::Type)
    return false;

  switch (actual) {
  case ImportKind::Module:
    llvm_unreachable("module imports do not bring in decls");
  case ImportKind::Type:
    llvm_unreachable("individual decls cannot have abstract import kind");
  case ImportKind::Struct:
  case ImportKind::Class:
  case ImportKind::Enum:
    return true;
  case ImportKind::Protocol:
  case ImportKind::Var:
  case ImportKind::Func:
    return false;
  }

  llvm_unreachable("Unhandled ImportKind in switch.");
}

static bool isNominalImportKind(ImportKind kind) {
  switch (kind) {
  case ImportKind::Module:
    llvm_unreachable("module imports do not bring in decls");
  case ImportKind::Struct:
  case ImportKind::Class:
  case ImportKind::Enum:
  case ImportKind::Protocol:
    return true;
  case ImportKind::Type:
  case ImportKind::Var:
  case ImportKind::Func:
    return false;
  }
  llvm_unreachable("unhandled kind");
}

static const char *getImportKindString(ImportKind kind) {
  switch (kind) {
  case ImportKind::Module:
    llvm_unreachable("module imports do not bring in decls");
  case ImportKind::Type:
    return "typealias";
  case ImportKind::Struct:
    return "struct";
  case ImportKind::Class:
    return "class";
  case ImportKind::Enum:
    return "enum";
  case ImportKind::Protocol:
    return "protocol";
  case ImportKind::Var:
    return "var";
  case ImportKind::Func:
    return "func";
  }

  llvm_unreachable("Unhandled ImportKind in switch.");
}

ArrayRef<ValueDecl *>
ScopedImportLookupRequest::evaluate(Evaluator &evaluator,
                                    ImportDecl *import) const {
  using namespace namelookup;

  auto importKind = import->getImportKind();
  assert(importKind != ImportKind::Module);

  // If we weren't able to load the module referenced by the import, we're done.
  // The fact that we failed to load the module has already been diagnosed by
  // import resolution.
  auto *module = import->getModule();
  if (!module)
    return ArrayRef<ValueDecl *>();

  /// Validate the scoped import.
  ///
  /// We validate the scope by making sure that the named declaration exists
  /// and is of the kind indicated by the keyword. This can't be done until
  /// we've performed import resolution, since that can introduce additional
  /// imports (such as cross-import overlays) which could provide the declaration.
  auto &ctx = module->getASTContext();
  auto declPath = import->getDeclPath();
  auto modulePath = import->getModulePath();
  auto *topLevelModule = module->getTopLevelModule();

  // Lookup the referenced decl in the top-level module. This is necessary as
  // the Clang importer currently handles submodules by importing their decls
  // into the top-level module.
  // FIXME: Doesn't handle scoped testable imports correctly.
  assert(declPath.size() == 1 && "can't handle sub-decl imports");
  SmallVector<ValueDecl *, 8> decls;
  lookupInModule(topLevelModule, declPath.front().Item, decls,
                 NLKind::QualifiedLookup, ResolutionKind::Overloadable,
                 import->getDeclContext()->getModuleScopeContext());

  auto importLoc = import->getLoc();
  if (decls.empty()) {
    ctx.Diags.diagnose(importLoc, diag::decl_does_not_exist_in_module,
                       static_cast<unsigned>(importKind), declPath.front().Item,
                       modulePath.front().Item)
      .highlight(SourceRange(declPath.front().Loc, declPath.back().Loc));
    return ArrayRef<ValueDecl *>();
  }

  Optional<ImportKind> actualKind = ImportDecl::findBestImportKind(decls);
  if (!actualKind.hasValue()) {
    // FIXME: print entire module name?
    ctx.Diags.diagnose(importLoc, diag::ambiguous_decl_in_module,
                       declPath.front().Item, module->getName());
    for (auto next : decls)
      ctx.Diags.diagnose(next, diag::found_candidate);

  } else if (!isCompatibleImportKind(importKind, *actualKind)) {
    Optional<InFlightDiagnostic> emittedDiag;
    if (*actualKind == ImportKind::Type && isNominalImportKind(importKind)) {
      assert(decls.size() == 1 &&
             "if we start suggesting ImportKind::Type for, e.g., a mix of "
             "structs and classes, we'll need a different message here");
      assert(isa<TypeAliasDecl>(decls.front()) &&
             "ImportKind::Type is only the best choice for a typealias");
      auto *typealias = cast<TypeAliasDecl>(decls.front());
      emittedDiag.emplace(ctx.Diags.diagnose(
          importLoc, diag::imported_decl_is_wrong_kind_typealias,
          typealias->getDescriptiveKind(),
          TypeAliasType::get(typealias, Type(), SubstitutionMap(),
                             typealias->getUnderlyingType()),
          getImportKindString(importKind)));
    } else {
      emittedDiag.emplace(ctx.Diags.diagnose(
          importLoc, diag::imported_decl_is_wrong_kind, declPath.front().Item,
          getImportKindString(importKind), static_cast<unsigned>(*actualKind)));
    }

    emittedDiag->fixItReplace(SourceRange(import->getKindLoc()),
                              getImportKindString(*actualKind));
    emittedDiag->flush();

    if (decls.size() == 1)
      ctx.Diags.diagnose(decls.front(), diag::decl_declared_here,
                         decls.front()->getName());
  }
  return ctx.AllocateCopy(decls);
}

//===----------------------------------------------------------------------===//
// MARK: Cross-import overlays
//===----------------------------------------------------------------------===//

static bool canCrossImport(const ImportedModuleDesc &import) {
  if (import.importOptions.contains(ImportFlags::Testable))
    return false;
  if (import.importOptions.contains(ImportFlags::PrivateImport))
    return false;

  return true;
}

/// Create an UnboundImport for a cross-import overlay.
UnboundImport::UnboundImport(ASTContext &ctx, const UnboundImport &base,
                             Identifier overlayName,
                             const ImportedModuleDesc &declaringImport,
                             const ImportedModuleDesc &bystandingImport)
    : importLoc(base.importLoc), options(), privateImportFileName(),
      modulePath(),
      // If the declaring import was scoped, inherit that scope in the
      // overlay's import.
      declPath(declaringImport.module.accessPath),
      importOrUnderlyingModuleDecl(declaringImport.module.importedModule) {
  modulePath = ctx.AllocateCopy(
      ModuleDecl::AccessPathTy( { overlayName, base.modulePath[0].Loc }));

  // A cross-import is never private or testable, and never comes from a private
  // or testable import.
  assert(canCrossImport(declaringImport));
  assert(canCrossImport(bystandingImport));

  auto &declaringOptions = declaringImport.importOptions;
  auto &bystandingOptions = bystandingImport.importOptions;

  // If both are exported, the cross-import is exported.
  if (declaringOptions.contains(ImportFlags::Exported) &&
      bystandingOptions.contains(ImportFlags::Exported))
    options |= ImportFlags::Exported;

  // If either are implementation-only, the cross-import is
  // implementation-only.
  if (declaringOptions.contains(ImportFlags::ImplementationOnly) ||
      bystandingOptions.contains(ImportFlags::ImplementationOnly))
    options |= ImportFlags::ImplementationOnly;
}

void ImportResolver::crossImport(ModuleDecl *M, UnboundImport &I) {
  // FIXME: There is a fundamental problem with this find-as-we-go approach:
  // The '@_exported import'-ed modules in this module's other files should be
  // taken into account, but they haven't been bound yet, and binding them would
  // require cross-importing. Chicken, meet egg.
  //
  // The way to fix this is probably to restructure import resolution so we
  // first bind all exported imports in all files, then bind all other imports
  // in each file. This may become simpler if we bind all ImportDecls before we
  // start computing cross-imports, but I haven't figured that part out yet.
  //
  // Fixing this is tracked within Apple by rdar://problem/59527118. I haven't
  // filed an SR because I plan to address it myself, but if this comment is
  // still here in April 2020 without an SR number, please file a Swift bug and
  // harass @brentdax to fill in the details.

  if (!SF.shouldCrossImport())
    return;

  if (I.getUnderlyingModule()) {
    auto underlying = I.getUnderlyingModule().get();

    // If this is a clang module, and it has a clang overlay, we want the
    // separately-imported overlay to sit on top of the clang overlay.
    if (underlying->isNonSwiftModule())
      underlying = underlying->getTopLevelModule(true);

    // FIXME: Should we warn if M doesn't reexport underlyingModule?
    SF.addSeparatelyImportedOverlay(M, underlying);
  }

  auto newImports = crossImportableModules.getArrayRef()
                        .drop_front(nextModuleToCrossImport);

  if (newImports.empty())
    // Nothing to do except crash when we read past the end of
    // crossImportableModules in that assert at the bottom.
    return;

  for (auto &newImport : newImports) {
    if (!canCrossImport(newImport))
      continue;

    // First we check if any of the imports of modules that have declared
    // cross-imports have declared one with this module.
    findCrossImportsInLists(I, crossImportDeclaringModules, {newImport},
                            /*shouldDiagnoseRedundantCrossImports=*/false);

    // If this module doesn't declare any cross-imports, we're done with this
    // import.
    if (!newImport.module.importedModule->mightDeclareCrossImportOverlays())
      continue;

    // Fine, we need to do the slow-but-rare thing: check if this import
    // declares a cross-import with any previous one.
    auto oldImports =
        // Slice from the start of crossImportableModules up to newImport.
        llvm::makeArrayRef(crossImportableModules.getArrayRef().data(),
                           &newImport);
    findCrossImportsInLists(I, {newImport}, oldImports,
                            /*shouldDiagnoseRedundantCrossImports=*/true);

    // Add this to the list of imports everyone needs to check against.
    crossImportDeclaringModules.push_back(newImport);
  }

  // Catch potential memory smashers
  assert(newImports.data() ==
             &crossImportableModules[nextModuleToCrossImport] &&
         "findCrossImports() should never mutate visibleModules");

  nextModuleToCrossImport = crossImportableModules.size();
}

void ImportResolver::findCrossImportsInLists(
    UnboundImport &I, ArrayRef<ImportedModuleDesc> declaring,
    ArrayRef<ImportedModuleDesc> bystanding,
    bool shouldDiagnoseRedundantCrossImports) {
  for (auto &declaringImport : declaring) {
    if (!canCrossImport(declaringImport))
      continue;

    for (auto &bystandingImport : bystanding) {
      if (!canCrossImport(bystandingImport))
        continue;

      findCrossImports(I, declaringImport, bystandingImport,
                       shouldDiagnoseRedundantCrossImports);
    }
  }
}

void ImportResolver::findCrossImports(
    UnboundImport &I, const ImportedModuleDesc &declaringImport,
    const ImportedModuleDesc &bystandingImport,
    bool shouldDiagnoseRedundantCrossImports) {
  assert(&declaringImport != &bystandingImport);

  LLVM_DEBUG(llvm::dbgs() << "Discovering cross-imports for '"
                          << declaringImport.module.importedModule->getName()
                          << "' -> '"
                          << bystandingImport.module.importedModule->getName()
                          << "'\n");

  if (ctx.Stats)
    ctx.Stats->getFrontendCounters().NumCrossImportsChecked++;

  // Find modules we need to import.
  SmallVector<Identifier, 4> names;
  declaringImport.module.importedModule->findDeclaredCrossImportOverlays(
      bystandingImport.module.importedModule->getName(), names, I.importLoc);

  // If we're diagnosing cases where we cross-import in both directions, get the
  // inverse list. Otherwise, leave the list empty.
  SmallVector<Identifier, 4> oppositeNames;
  if (shouldDiagnoseRedundantCrossImports)
    bystandingImport.module.importedModule->findDeclaredCrossImportOverlays(
        declaringImport.module.importedModule->getName(), oppositeNames,
        I.importLoc);

  if (ctx.Stats && !names.empty())
    ctx.Stats->getFrontendCounters().NumCrossImportsFound++;

  // Add import statements.
  for (auto &name : names) {
    // If we are actually compiling part of this overlay, don't try to load the
    // overlay.
    if (name == SF.getParentModule()->getName())
      continue;

    unboundImports.emplace_back(
        declaringImport.module.importedModule->getASTContext(), I, name,
        declaringImport, bystandingImport);

    if (llvm::is_contained(oppositeNames, name))
      ctx.Diags.diagnose(I.importLoc, diag::cross_imported_by_both_modules,
                         declaringImport.module.importedModule->getName(),
                         bystandingImport.module.importedModule->getName(),
                         name);

    if (ctx.LangOpts.EnableCrossImportRemarks)
      ctx.Diags.diagnose(I.importLoc, diag::cross_import_added,
                         declaringImport.module.importedModule->getName(),
                         bystandingImport.module.importedModule->getName(),
                         name);

    LLVM_DEBUG({
      auto &crossImportOptions = unboundImports.back().options;
      llvm::dbgs() << "  ";
      if (crossImportOptions.contains(ImportFlags::Exported))
        llvm::dbgs() << "@_exported ";
      if (crossImportOptions.contains(ImportFlags::ImplementationOnly))
        llvm::dbgs() << "@_implementationOnly ";
      llvm::dbgs() << "import " << name << "\n";
    });
  }
}

static bool isSubmodule(ModuleDecl* M) {
  auto clangMod = M->findUnderlyingClangModule();
  return clangMod && clangMod->Parent;
}

void ImportResolver::addCrossImportableModules(ImportedModuleDesc importDesc) {
  // FIXME: namelookup::getAllImports() doesn't quite do what we need (mainly
  // w.r.t. scoped imports), but it seems like we could extend it to do so, and
  // then eliminate most of this.

  SmallVector<ImportedModule, 16> importsWorklist = { importDesc.module };

  while (!importsWorklist.empty()) {
    auto nextImport = importsWorklist.pop_back_val();

    // If they are both scoped, and they are *differently* scoped, this import
    // cannot possibly expose anything new. Skip it.
    if (!importDesc.module.accessPath.empty() &&
        !nextImport.accessPath.empty() &&
        !ModuleDecl::isSameAccessPath(importDesc.module.accessPath,
                                      nextImport.accessPath))
      continue;

    // If we are importing a submodule, treat it as though we imported its
    // top-level module (or rather, the top-level module's clang overlay if it
    // has one).
    if (isSubmodule(nextImport.importedModule)) {
      nextImport.importedModule =
          nextImport.importedModule->getTopLevelModule(/*overlay=*/true);

      // If the rewritten import is now for our own parent module, this was an
      // import of our own clang submodule in a mixed-language module. We don't
      // want to process our own cross-imports.
      if (nextImport.importedModule == SF.getParentModule())
        continue;
    }

    // Drop this module into the ImportDesc so we treat it as imported with the
    // same options and scope as `I`.
    importDesc.module.importedModule = nextImport.importedModule;

    // Add it to the list of cross-importable modules. If it's already there,
    // we've already done the rest of the work of this loop iteration and can
    // skip it.
    if (!crossImportableModules.insert(importDesc))
      continue;

    // We don't consider the re-exports of ObjC modules because ObjC re-exports
    // everything, so there isn't enough signal there to work from.
    if (nextImport.importedModule->isNonSwiftModule())
      continue;

    // Add the module's re-exports to worklist.
    nextImport.importedModule->getImportedModules(
        importsWorklist, ModuleDecl::ImportFilterKind::Public);
  }
}

LLVM_ATTRIBUTE_USED static void dumpCrossImportOverlays(ModuleDecl* M) {
  llvm::dbgs() << "'" << M->getName() << "' declares cross-imports with bystanders:\n";

  SmallVector<Identifier, 4> secondaries;
  M->getDeclaredCrossImportBystanders(secondaries);

  for (auto secondary : secondaries)
    llvm::dbgs() << "  " << secondary << "\n";
}


text:
const config = require('@cheevr/config').addDefaultConfig(__dirname, 'config');
const Database = require('@cheevr/database');
const EventEmitter = require('events').EventEmitter;
const fs = require('fs');
const Logger = require('@cheevr/logging');
const path = require('path');
const Task = require('./task');

const cwd = process.cwd();
const log = Logger[config.tasks.logger];

/**
 * Holds information about all the tasks. The hierarchy is as follows:
 *
 * TaskManager > Tasks      The task manager holds all top level tasks
 * Task > Workers           Each tasks can be executed in parallel with multiple workers
 * Worker <> Runner         Each worker has one runner that it communicates with
 * Runner > Jobs            Each runner can execute multiple jobs at independent intervals (depending on the type of job)
 */
class TaskManager extends EventEmitter {
    constructor() {
        super();
        Database.ready && this.reload() || Database.on('ready', () => this.reload());
    }

    /**
     * This will kill off any running tasks and relaunch them with the current state of the configuration.
     */
    reload() {
        for (let file in this._tasks) {
            this._tasks[file].kill();
        }
        this._tasks = {};
        let taskDirs = config.normalizePath(cwd, config.paths.tasks);
        Database.ready && this.scanPath(taskDirs);
    }

    /**
     * Looks at the given path for javascript files to include and creates task processes for them.
     * @param {string|string[]} taskPath    The file or directory to create a task off of
     */
    scanPath(taskPath) {
        if (!config.tasks.enabled) {
            return;
        }
        taskPath = Array.isArray(taskPath) ? taskPath : [ taskPath ];
        for (let dir of taskPath) {
            if (fs.existsSync(dir)) {
                if (fs.statSync(dir).isDirectory()) {
                    let files = fs.readdirSync(dir);
                    for (let file of files) {
                        let fullPath = path.join(dir, file);
                        this.addTask(fullPath);
                    }
                } else {
                    this.addTask(dir);
                }
            }
        }
    }

    /**
     * Will create a new task with one or more associated runners that will execute the jobs inside the task.
     * @param {string} file The task file that should be added.
     * @returns {Task} A task object that allows to send messages to one or more workers
     */
    async addTask(file) {
        if (!config.tasks.enabled) {
            return;
        }
        await new Promise(resolve => Database.ready && resolve() || Database.on('ready', resolve));
        file = path.normalize(file);
        let task = this._tasks[file];
        if (!task) {
            this._tasks[file] = task = new Task(file);
        }
        await task.ready();
        this.emit('added', task);
        return task;
    }

    /**
     * Sets up a status api for tasks.
     * @param {express.Router} router   The express router (or a compatible implementation)
     * @param {string} [urlPath]        The path prefix under which the status api should be accessible
     */
    endpoint(router, urlPath = config.tasks.urlPath) {
        if (config.tasks.enabled) {
            router.get(urlPath + '/status', (req, res) => {
                let response = {};
                for (let file in this._tasks) {
                    let task = this._tasks[file];
                    let workers = {};
                    for (let worker of task.workers) {
                        // TODO workers are set to be enabled:false (when they are not)
                        workers[worker.id] = worker.state;
                    }
                    response[task.name] = {
                        workers,
                        enabled: task.enabled
                    }
                }
                res.jsonp(response).end();
            });
            router.get(urlPath + '/:taskName/enable', (req, res, next) => {
                for (let file in this._tasks) {
                    if (file.endsWith(req.params.taskName)) {
                        this._tasks[file].enabled = true;
                        log.info('Set task "%s" to be enabled', req.params.taskName);
                        return res.status(204).end();
                    }
                }
                next();
            });
            router.get(urlPath + '/:taskName/disable', (req, res, next) => {
                for (let file in this._tasks) {
                    if (file.endsWith(req.params.taskName)) {
                        log.info('Set task "%s" to be disabled', req.params.taskName);
                        this._tasks[file].enabled = false;
                        return res.status(204).end();
                    }
                }
                next();
            });
            router.get(urlPath + '/:taskName/workers/:count', (req, res, next) => {
                for (let file in this._tasks) {
                    if (file.endsWith(req.params.taskName)) {
                        log.info('Set number of workers for task "%s" to %s', req.params.taskName, req.params.count);
                        this._tasks[file].workersWanted = req.params.count;
                        return res.status(204).end();
                    }
                }
                next();
            });
            router.get(urlPath + '/:taskName/trigger/:jobName', (req, res, next) => {
                for (let file in this._tasks) {
                    if (file.endsWith(req.params.taskName)) {
                        this._tasks[file].roundRobin._runJob(req.params.jobName);
                        log.info('Manually triggered job "%s" in task "%s"', req.params.jobName, req.params.taskName);
                        return res.status(204).end();
                    }
                }
                next();
            });
        }
    }

    /**
     * Returns an array of all workers for a given tasks file.
     * @param {string} file The filename (same as task.id) that identifies this task.
     * @returns {Task}
     */
    task(file) {
        return this._tasks[file];
    }
}

module.exports = exports = new TaskManager();

redpajama_set_name: RedPajamaGithub


text:
A convenient ReactiveVar-like wrapper for session variables. This project contains the source for the [Meteor](https://www.meteor.com/) package session-var.

See the package documentation [here](packages/session-var/README.md).

# Install

  	meteor add dschnare:session-var


text:
"""TensorFlow estimators for Linear and DNN joined training models."""

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import inspect
import math

import numpy as np
import six

from tensorflow.contrib import layers
from tensorflow.contrib import metrics as metrics_lib
from tensorflow.contrib.framework.python.ops import variables as contrib_variables
from tensorflow.contrib.learn.python.learn.estimators import estimator
from tensorflow.python.framework import dtypes
from tensorflow.python.framework import ops
from tensorflow.python.ops import array_ops
from tensorflow.python.ops import control_flow_ops
from tensorflow.python.ops import gradients
from tensorflow.python.ops import logging_ops
from tensorflow.python.ops import math_ops
from tensorflow.python.ops import nn
from tensorflow.python.ops import parsing_ops
from tensorflow.python.ops import state_ops
from tensorflow.python.ops import variables
from tensorflow.python.training import training

# TODO(ispir): Increase test coverage
class _DNNLinearCombinedBaseEstimator(estimator.BaseEstimator):
  """An estimator for TensorFlow Linear and DNN joined training models.

    Input of `fit`, `train`, and `evaluate` should have following features,
      otherwise there will be a `KeyError`:
        if `weight_column_name` is not `None`, a feature with
          `key=weight_column_name` whose value is a `Tensor`.
        for each `column` in `dnn_feature_columns` + `linear_feature_columns`:
        - if `column` is a `SparseColumn`, a feature with `key=column.name`
          whose `value` is a `SparseTensor`.
        - if `column` is a `RealValuedColumn, a feature with `key=column.name`
          whose `value` is a `Tensor`.

  Parameters:
    model_dir: Directory to save model parameters, graph and etc.
    n_classes: number of target classes. Default is binary classification.
    weight_column_name: A string defining feature column name representing
      weights. It is used to down weight or boost examples during training. It
      will be multiplied by the loss of the example.
    linear_feature_columns: An iterable containing all the feature columns used
      by linear part of the model. All items in the set should be instances of
      classes derived from `FeatureColumn`.
    linear_optimizer: An instance of `tf.Optimizer` used to apply gradients to
      the linear part of the model. If `None`, will use a FTRL optimizer.
    dnn_feature_columns: An iterable containing all the feature columns used by
      deep part of the model. All items in the set should be instances of
      classes derived from `FeatureColumn`.
    dnn_hidden_units: List of hidden units per layer. All layers are fully
      connected.
    dnn_optimizer: An instance of `tf.Optimizer` used to apply gradients to the
      deep part of the model. If `None`, will use an Adagrad optimizer.
    dnn_activation_fn: Activation function applied to each layer. If `None`,
      will use `tf.nn.relu`.
    dnn_dropout: When not None, the probability we will drop out
      a given coordinate.
    config: RunConfig object to configure the runtime settings.

    Raises:
      ValueError: If both linear_feature_columns and dnn_features_columns are
        empty at the same time.
  """

  def __init__(self,
               model_dir=None,
               weight_column_name=None,
               linear_feature_columns=None,
               linear_optimizer=None,
               dnn_feature_columns=None,
               dnn_optimizer=None,
               dnn_hidden_units=None,
               dnn_activation_fn=nn.relu,
               dnn_dropout=None,
               config=None):
    super(_DNNLinearCombinedBaseEstimator, self).__init__(model_dir=model_dir,
                                                          config=config)
    self._weight_column_name = weight_column_name
    self._linear_feature_columns = linear_feature_columns
    self._linear_optimizer = linear_optimizer
    self._dnn_feature_columns = dnn_feature_columns
    self._dnn_optimizer = dnn_optimizer
    self._dnn_hidden_units = dnn_hidden_units
    self._dnn_activation_fn = dnn_activation_fn
    if self._dnn_activation_fn is None:
      self._dnn_activation_fn = nn.relu
    self._dnn_dropout = dnn_dropout
    self._dnn_weight_collection = "DNNLinearCombined_dnn"
    self._linear_weight_collection = "DNNLinearCombined_linear"
    self._centered_bias_weight_collection = "centered_bias"

  @property
  def linear_weights_(self):
    """Returns weights per feature of the linear part."""
    all_variables = self.get_variable_names()
    # TODO(ispir): Figure out a better way to retrieve variables for features.
    # for example using feature info / columns.
    values = {}
    for name in all_variables:
      if (name.startswith("linear/") and name.rfind("/") == 6 and
          name != "linear/bias_weight"):
        values[name] = self.get_variable_value(name)
    if len(values) == 1:
      return values[list(values.keys())[0]]
    return values

  @property
  def linear_bias_(self):
    """Returns bias of the linear part."""
    return (self.get_variable_value("linear/bias_weight") +
            self.get_variable_value("centered_bias_weight"))

  @property
  def dnn_weights_(self):
    """Returns weights of deep neural network part."""
    return [self.get_variable_value("hiddenlayer_%d/weights" % i)
            for i, _ in enumerate(self._dnn_hidden_units)] + [
                self.get_variable_value("dnn_logit/weights")]

  @property
  def dnn_bias_(self):
    """Returns bias of deep neural network part."""
    return [self.get_variable_value("hiddenlayer_%d/bias" % i)
            for i, _ in enumerate(self._dnn_hidden_units)] + [
                self.get_variable_value("dnn_logit/bias"),
                self.get_variable_value("centered_bias_weight")]

  def _get_train_ops(self, features, targets):
    """See base class."""
    global_step = contrib_variables.get_global_step()
    assert global_step
    logits = self._logits(features, is_training=True)
    with ops.control_dependencies([self._centered_bias_step(
        targets, self._get_weight_tensor(features))]):
      loss = self._loss(logits, targets, self._get_weight_tensor(features))
    logging_ops.scalar_summary("loss", loss)

    linear_vars = self._get_linear_vars()
    dnn_vars = self._get_dnn_vars()
    grads = gradients.gradients(loss, dnn_vars + linear_vars)
    dnn_grads = grads[0:len(dnn_vars)]
    linear_grads = grads[len(dnn_vars):]

    train_ops = self._get_linear_training_ops(
        linear_grads, linear_vars) + self._get_dnn_training_ops(dnn_grads,
                                                                dnn_vars)

    train_step = control_flow_ops.group(*train_ops, name="combined_training_op")
    with ops.control_dependencies([train_step]):
      with ops.get_default_graph().colocate_with(global_step):
        return state_ops.assign_add(global_step, 1).op, loss

  def _run_metrics(self, predictions, targets, metrics, weights):
    result = {}
    targets = math_ops.cast(targets, predictions.dtype)
    for name, metric in six.iteritems(metrics or {}):
      if "weights" in inspect.getargspec(metric)[0]:
        result[name] = metric(predictions, targets, weights=weights)
      else:
        result[name] = metric(predictions, targets)

    return result

  def _get_eval_ops(self, features, targets, metrics=None):
    raise NotImplementedError

  def _get_predict_ops(self, features):
    """See base class."""
    logits = self._logits(features)
    return self._logits_to_predictions(logits, proba=True)

  def _logits_to_predictions(self, logits, proba=False):
    raise NotImplementedError

  def _get_feature_ops_from_example(self, examples_batch):
    column_types = layers.create_feature_spec_for_parsing((
        self._get_linear_feature_columns() or []) + (
            self._get_dnn_feature_columns() or []))
    features = parsing_ops.parse_example(examples_batch, column_types)
    return features

  def _num_label_columns(self):
    raise NotImplementedError

  def _get_linear_feature_columns(self):
    return sorted(
        set(self._linear_feature_columns),
        key=lambda x: x.key) if self._linear_feature_columns else None

  def _get_dnn_feature_columns(self):
    return sorted(set(
        self._dnn_feature_columns)) if self._dnn_feature_columns else None

  def _dnn_logits(self, features, is_training=False):
    net = layers.input_from_feature_columns(
        features,
        self._get_dnn_feature_columns(),
        weight_collections=[self._dnn_weight_collection])
    for layer_id, num_hidden_units in enumerate(self._dnn_hidden_units):
      net = layers.legacy_fully_connected(
          net,
          num_hidden_units,
          activation_fn=self._dnn_activation_fn,
          weight_collections=[self._dnn_weight_collection],
          bias_collections=[self._dnn_weight_collection],
          name="hiddenlayer_%d" % layer_id)
      if self._dnn_dropout is not None and is_training:
        net = layers.dropout(
            net,
            keep_prob=(1.0 - self._dnn_dropout))
      self._add_hidden_layer_summary(net, "hiddenlayer_%d" % layer_id)
    logit = layers.legacy_fully_connected(
        net,
        self._num_label_columns(),
        weight_collections=[self._dnn_weight_collection],
        bias_collections=[self._dnn_weight_collection],
        name="dnn_logit")
    self._add_hidden_layer_summary(logit, "dnn_logit")
    return logit

  def _add_hidden_layer_summary(self, value, tag):
    # TODO(zakaria): Move this code to tf.learn and add test.
    logging_ops.scalar_summary("%s:fraction_of_zero_values" % tag,
                               nn.zero_fraction(value))
    logging_ops.histogram_summary("%s:activation" % tag, value)

  def _linear_logits(self, features):
    logits, _, _ = layers.weighted_sum_from_feature_columns(
        columns_to_tensors=features,
        feature_columns=self._get_linear_feature_columns(),
        num_outputs=self._num_label_columns(),
        weight_collections=[self._linear_weight_collection],
        name="linear")
    return logits

  def _get_feature_dict(self, features):
    if isinstance(features, dict):
      return features
    return {"": features}

  def _centered_bias(self):
    centered_bias = variables.Variable(
        array_ops.zeros([self._num_label_columns()]),
        collections=[self._centered_bias_weight_collection,
                     ops.GraphKeys.VARIABLES],
        name="centered_bias_weight")
    logging_ops.scalar_summary(
        ["centered_bias_%d" % cb for cb in range(self._num_label_columns())],
        array_ops.reshape(centered_bias, [-1]))
    return centered_bias

  def _centered_bias_step(self, targets, weight_tensor):
    centered_bias = ops.get_collection(self._centered_bias_weight_collection)
    batch_size = array_ops.shape(targets)[0]
    logits = array_ops.reshape(
        array_ops.tile(centered_bias[0], [batch_size]),
        [batch_size, self._num_label_columns()])
    loss = self._loss(logits, targets, weight_tensor)
    # Learn central bias by an optimizer. 0.1 is a convervative lr for a single
    # variable.
    return training.AdagradOptimizer(0.1).minimize(loss, var_list=centered_bias)

  def _logits(self, features, is_training=False):
    if not (self._get_linear_feature_columns() or
            self._get_dnn_feature_columns()):
      raise ValueError("Either linear_feature_columns or dnn_feature_columns "
                       "should be defined.")

    features = self._get_feature_dict(features)
    if self._get_linear_feature_columns() and self._get_dnn_feature_columns():
      logits = (self._linear_logits(features) +
                self._dnn_logits(features, is_training=is_training))
    elif self._get_dnn_feature_columns():
      logits = self._dnn_logits(features, is_training=is_training)
    else:
      logits = self._linear_logits(features)

    return nn.bias_add(logits, self._centered_bias())

  def _get_weight_tensor(self, features):
    if not self._weight_column_name:
      return None
    else:
      return array_ops.reshape(
          math_ops.to_float(features[self._weight_column_name]),
          shape=(-1,))

  def _loss_vec(self, logits, targets):
    raise NotImplementedError

  def _loss(self, logits, target, weight_tensor):
    loss_vec = self._loss_vec(logits, target)

    if weight_tensor is None:
      return math_ops.reduce_mean(loss_vec, name="loss")
    else:
      loss_vec = array_ops.reshape(loss_vec, shape=(-1,))
      loss_vec = math_ops.mul(
          loss_vec, array_ops.reshape(weight_tensor, shape=(-1,)))
      return math_ops.div(
          math_ops.reduce_sum(loss_vec),
          math_ops.to_float(math_ops.reduce_sum(weight_tensor)),
          name="loss")

  def _get_linear_vars(self):
    if self._get_linear_feature_columns():
      return ops.get_collection(self._linear_weight_collection)
    return []

  def _get_linear_training_ops(self, linear_grads, linear_vars):
    if self._get_linear_feature_columns():
      self._linear_optimizer = self._get_optimizer(
          self._linear_optimizer,
          default_optimizer="Ftrl",
          default_learning_rate=1. / math.sqrt(len(
              self._get_linear_feature_columns())))
      return [
          self._linear_optimizer.apply_gradients(zip(linear_grads, linear_vars))
      ]
    return []

  def _get_dnn_vars(self):
    if self._get_dnn_feature_columns():
      return ops.get_collection(self._dnn_weight_collection)
    return []

  def _get_dnn_training_ops(self, dnn_grads, dnn_vars):
    if self._get_dnn_feature_columns():
      self._dnn_optimizer = self._get_optimizer(self._dnn_optimizer,
                                                default_optimizer="Adagrad",
                                                default_learning_rate=0.05)
      return [self._dnn_optimizer.apply_gradients(zip(dnn_grads, dnn_vars))]
    return []

  def _get_optimizer(self, optimizer, default_optimizer, default_learning_rate):
    if optimizer is None:
      optimizer = default_optimizer
    if isinstance(optimizer, six.string_types):
      optimizer = layers.OPTIMIZER_CLS_NAMES[optimizer](
          learning_rate=default_learning_rate)
    return optimizer

class DNNLinearCombinedClassifier(_DNNLinearCombinedBaseEstimator):
  """A classifier for TensorFlow Linear and DNN joined training models.

  Example:
    ```
    installed_app_id = sparse_column_with_hash_bucket("installed_id", 1e6)
    impression_app_id = sparse_column_with_hash_bucket("impression_id", 1e6)

    installed_x_impression = crossed_column(
        [installed_app_id, impression_app_id])

    installed_emb = embedding_column(installed_app_id, dimension=16,
                                     combiner="sum")
    impression_emb = embedding_column(impression_app_id, dimension=16,
                                      combiner="sum")

    estimator = DNNLinearCombinedClassifier(
        # common settings
        n_classes, weight_column_name,
        # wide settings
        linear_feature_columns=[installed_x_impression],
        linear_optimizer=tf.train.FtrlOptimizer(...),
        # deep settings
        dnn_feature_columns=[installed_emb, impression_emb],
        dnn_hidden_units=[1000, 500, 100],
        dnn_optimizer=tf.train.AdagradOptimizer(...))

    # Input builders
    def input_fn_train: # returns X, Y
      ...
    def input_fn_eval: # returns X, Y
      ...
    estimator.train(input_fn_train)
    estimator.evaluate(input_fn_eval)
    estimator.predict(x)
    ```

    Input of `fit`, `train`, and `evaluate` should have following features,
      otherwise there will be a `KeyError`:
        if `weight_column_name` is not `None`, a feature with
          `key=weight_column_name` whose value is a `Tensor`.
        for each `column` in `dnn_feature_columns` + `linear_feature_columns`:
        - if `column` is a `SparseColumn`, a feature with `key=column.name`
          whose `value` is a `SparseTensor`.
        - if `column` is a `RealValuedColumn, a feature with `key=column.name`
          whose `value` is a `Tensor`.

  Parameters:
    model_dir: Directory to save model parameters, graph and etc.
    n_classes: number of target classes. Default is binary classification.
    weight_column_name: A string defining feature column name representing
      weights. It is used to down weight or boost examples during training. It
      will be multiplied by the loss of the example.
    linear_feature_columns: An iterable containing all the feature columns used
      by linear part of the model. All items in the set must be instances of
      classes derived from `FeatureColumn`.
    linear_optimizer: An instance of `tf.Optimizer` used to apply gradients to
      the linear part of the model. If `None`, will use a FTRL optimizer.
    dnn_feature_columns: An iterable containing all the feature columns used by
      deep part of the model. All items in the set must be instances of
      classes derived from `FeatureColumn`.
    dnn_hidden_units: List of hidden units per layer. All layers are fully
      connected.
    dnn_optimizer: An instance of `tf.Optimizer` used to apply gradients to the
      deep part of the model. If `None`, will use an Adagrad optimizer.
    dnn_activation_fn: Activation function applied to each layer. If `None`,
      will use `tf.nn.relu`.
    dnn_dropout: When not None, the probability we will drop out
      a given coordinate.
    config: RunConfig object to configure the runtime settings.

    Raises:
      ValueError: If both linear_feature_columns and dnn_features_columns are
        empty at the same time.
      ValueError: If both n_classes < 2.
  """

  def __init__(self,
               model_dir=None,
               n_classes=2,
               weight_column_name=None,
               linear_feature_columns=None,
               linear_optimizer=None,
               dnn_feature_columns=None,
               dnn_optimizer=None,
               dnn_hidden_units=None,
               dnn_activation_fn=nn.relu,
               dnn_dropout=None,
               config=None):

    if n_classes < 2:
      raise ValueError("n_classes should be greater than 1. Given: {}".format(
          n_classes))

    self._n_classes = n_classes
    super(DNNLinearCombinedClassifier, self).__init__(
        model_dir=model_dir,
        weight_column_name=weight_column_name,
        linear_feature_columns=linear_feature_columns,
        linear_optimizer=linear_optimizer,
        dnn_feature_columns=dnn_feature_columns,
        dnn_optimizer=dnn_optimizer,
        dnn_hidden_units=dnn_hidden_units,
        dnn_activation_fn=dnn_activation_fn,
        dnn_dropout=dnn_dropout,
        config=config)

  def predict(self, x=None, input_fn=None, batch_size=None):
    """Returns predictions for given features.

    Args:
      x: features.
      input_fn: Input function. If set, x must be None.
      batch_size: Override default batch size.

    Returns:
      Numpy array of predicted classes or regression values.
    """
    predictions = super(DNNLinearCombinedClassifier, self).predict(
        x=x, input_fn=input_fn, batch_size=batch_size)
    predictions = np.argmax(predictions, axis=1)
    return predictions

  def predict_proba(self, x=None, input_fn=None, batch_size=None):
    """Returns prediction probabilities for given features.

    Args:
      x: features.
      input_fn: Input function. If set, x and y must be None.
      batch_size: Override default batch size.

    Returns:
      Numpy array of predicted probabilities.
    """
    return super(DNNLinearCombinedClassifier, self).predict(
        x=x, input_fn=input_fn, batch_size=batch_size)

  def _loss_vec(self, logits, target):
    # Check that we got int32/int64 for classification.
    if (not target.dtype.is_compatible_with(dtypes.int64) and
        not target.dtype.is_compatible_with(dtypes.int32)):
      raise ValueError("Target's dtype should be int32, int64 or compatible. "
                       "Instead got %s." % target.dtype)

    if self._n_classes == 2:
      # sigmoid_cross_entropy_with_logits requires [batch_size, 1] target.
      if len(target.get_shape()) == 1:
        target = array_ops.expand_dims(target, dim=[1])
      loss_vec = nn.sigmoid_cross_entropy_with_logits(
          logits, math_ops.to_float(target))
    else:
      # sparse_softmax_cross_entropy_with_logits requires [batch_size] target.
      if len(target.get_shape()) == 2:
        target = array_ops.squeeze(target, squeeze_dims=[1])
      loss_vec = nn.sparse_softmax_cross_entropy_with_logits(
          logits, target)
    return loss_vec

  def _logits_to_predictions(self, logits, proba=False):
    if self._n_classes == 2:
      logits = array_ops.concat(1, [array_ops.zeros_like(logits), logits])

    if proba:
      return nn.softmax(logits)
    else:
      return math_ops.argmax(logits, 1)

  def _num_label_columns(self):
    return 1 if self._n_classes == 2 else self._n_classes

  def _get_eval_ops(self, features, targets, metrics=None):
    """See base class."""
    logits = self._logits(features)
    result = {"loss": metrics_lib.streaming_mean(self._loss(
        logits, targets,
        weight_tensor=self._get_weight_tensor(features)))}

    # Adding default metrics
    if metrics is None:
      metrics = {"accuracy": metrics_lib.streaming_accuracy}

    if self._n_classes == 2:
      predictions = math_ops.sigmoid(logits)
      result["eval_auc"] = metrics_lib.streaming_auc(predictions, targets)

    if metrics:
      predictions = self._logits_to_predictions(logits, proba=False)
      result.update(self._run_metrics(predictions, targets, metrics,
                                      self._get_weight_tensor(features)))

    return result

class DNNLinearCombinedRegressor(_DNNLinearCombinedBaseEstimator):
  """A regressor for TensorFlow Linear and DNN joined training models.

  Example:
    ```
    installed_app_id = sparse_column_with_hash_bucket("installed_id", 1e6)
    impression_app_id = sparse_column_with_hash_bucket("impression_id", 1e6)

    installed_x_impression = crossed_column(
        [installed_app_id, impression_app_id])

    installed_emb = embedding_column(installed_app_id, dimension=16,
                                     combiner="sum")
    impression_emb = embedding_column(impression_app_id, dimension=16,
                                      combiner="sum")

    estimator = DNNLinearCombinedClassifier(
        # common settings
        n_classes, weight_column_name,
        # wide settings
        linear_feature_columns=[installed_x_impression],
        linear_optimizer=tf.train.FtrlOptimizer(...),
        # deep settings
        dnn_feature_columns=[installed_emb, impression_emb],
        dnn_hidden_units=[1000, 500, 100],
        dnn_optimizer=tf.train.AdagradOptimizer(...))

    # Input builders
    def input_fn_train: # returns X, Y
      ...
    def input_fn_eval: # returns X, Y
      ...
    estimator.train(input_fn_train)
    estimator.evaluate(input_fn_eval)
    estimator.predict(x)
    ```

    Input of `fit`, `train`, and `evaluate` should have following features,
      otherwise there will be a `KeyError`:
        if `weight_column_name` is not `None`, a feature with
          `key=weight_column_name` whose value is a `Tensor`.
        for each `column` in `dnn_feature_columns` + `linear_feature_columns`:
        - if `column` is a `SparseColumn`, a feature with `key=column.name`
          whose `value` is a `SparseTensor`.
        - if `column` is a `RealValuedColumn, a feature with `key=column.name`
          whose `value` is a `Tensor`.

  Parameters:
    model_dir: Directory to save model parameters, graph and etc.
    weight_column_name: A string defining feature column name representing
      weights. It is used to down weight or boost examples during training. It
      will be multiplied by the loss of the example.
    linear_feature_columns: An iterable containing all the feature columns used
      by linear part of the model. All items in the set must be instances of
      classes derived from `FeatureColumn`.
    linear_optimizer: An instance of `tf.Optimizer` used to apply gradients to
      the linear part of the model. If `None`, will use a FTRL optimizer.
    dnn_feature_columns: An iterable containing all the feature columns used by
      deep part of the model. All items in the set must be instances of
      classes derived from `FeatureColumn`.
    dnn_hidden_units: List of hidden units per layer. All layers are fully
      connected.
    dnn_optimizer: An instance of `tf.Optimizer` used to apply gradients to the
      deep part of the model. If `None`, will use an Adagrad optimizer.
    dnn_activation_fn: Activation function applied to each layer. If None, will
      use `tf.nn.relu`.
    dnn_dropout: When not None, the probability we will drop out
      a given coordinate.
    config: RunConfig object to configure the runtime settings.

    Raises:
      ValueError: If both linear_feature_columns and dnn_features_columns are
        empty at the same time.
  """

  def __init__(self,
               model_dir=None,
               weight_column_name=None,
               linear_feature_columns=None,
               linear_optimizer=None,
               dnn_feature_columns=None,
               dnn_optimizer=None,
               dnn_hidden_units=None,
               dnn_activation_fn=nn.relu,
               dnn_dropout=None,
               config=None):
    super(DNNLinearCombinedRegressor, self).__init__(
        model_dir=model_dir,
        weight_column_name=weight_column_name,
        linear_feature_columns=linear_feature_columns,
        linear_optimizer=linear_optimizer,
        dnn_feature_columns=dnn_feature_columns,
        dnn_optimizer=dnn_optimizer,
        dnn_hidden_units=dnn_hidden_units,
        dnn_activation_fn=dnn_activation_fn,
        dnn_dropout=dnn_dropout,
        config=config)

  def predict(self, x=None, input_fn=None, batch_size=None):
    """Returns predictions for given features.

    Args:
      x: features.
      input_fn: Input function. If set, x must be None.
      batch_size: Override default batch size.

    Returns:
      Numpy array of predicted classes or regression values.
    """
    return super(DNNLinearCombinedRegressor, self).predict(
        x=x, input_fn=input_fn, batch_size=batch_size)

  def _loss_vec(self, logits, target):
    # To prevent broadcasting inside "-".
    if len(target.get_shape()) == 1:
      target = array_ops.expand_dims(target, dim=[1])
    logits.get_shape().assert_is_compatible_with(target.get_shape())
    return math_ops.square(logits - math_ops.to_float(target))

  def _logits_to_predictions(self, logits, proba=False):
    # TODO(ispir): Add target column support.
    if self._targets_info is None or len(self._targets_info.shape) == 1:
      return array_ops.squeeze(logits, squeeze_dims=[1])
    return logits

  def _num_label_columns(self):
    # TODO(ispir): Add target column support.
    if self._targets_info is None or len(self._targets_info.shape) == 1:
      return 1
    return int(self._targets_info.shape[1])

  def _get_eval_ops(self, features, targets, metrics=None):
    """See base class."""
    logits = self._logits(features)
    result = {"loss": metrics_lib.streaming_mean(self._loss(
        logits, targets,
        weight_tensor=self._get_weight_tensor(features)))}

    # Adding default metrics
    if metrics:
      predictions = self._logits_to_predictions(logits, proba=False)
      result.update(self._run_metrics(predictions, targets, metrics,
                                      self._get_weight_tensor(features)))

    return result


text:
(function () {
  'use strict';

  /**
   * @ngdoc overview
   * @module mfw.business.actionstore
   * @name mfw.business.actionstore
   *
   * @requires flux
   *
   * @description
   * # Description
   *
   * This module provides a centralized control and notification system to handle all async processes (involving promises)
   * in your application.
   *
   * It relies on {@link https://github.com/christianalfoni/flux-angular flux-angular} as notification mechanism and storage.
   */
  var ActionStoreModule = angular.module('mfw.business.actionstore', [
    'flux'
  ]);

  /**
   * @constant
   * @type {string}
   * @description Action status *running*.
   */
  var STATUS_RUNNING = 'running';
  /**
   * @constant
   * @type {string}
   * @description Action status *finished*.
   */
  var STATUS_FINISHED = 'finished';
  /**
   * @constant
   * @type {string}
   * @description Action status *error*.
   */
  var STATUS_ERROR = 'error';

  /**
   * @constant
   * @type {string}
   * @description Prefix to be used in all `$mfwActionStore` fired events.
   * @private
   */
  var ACTION_PREFIX = 'mfw:Action.';

  /**
   * @typedef {Object} MfwAction
   * @property {String} id Action identifier
   * @property {Promise} promise The promise
   * @property {String} status Action status
   *
   * @see STATUS_RUNNING
   * @see STATUS_FINISHED
   * @see STATUS_ERROR
   */

  /**
   * @ngdoc event
   * @name mfw.business.actionstore.service:$mfwActionStore#mfw:Action.<actionId>.<status>
   * @eventOf mfw.business.actionstore.service:$mfwActionStore
   * @eventType broadcast on $mfwActionStore
   *
   * @description
   * Event triggered when an action is updated at any status step.
   *
   * @param {MfwAction} action Updated action.
   */
  /**
   * @ngdoc event
   * @name mfw.business.actionstore.service:$mfwActionStore#mfw:Action.<status>.<actionId>
   * @eventOf mfw.business.actionstore.service:$mfwActionStore
   * @eventType broadcast on $mfwActionStore
   *
   * @description
   * Event triggered when an action is updated at a specific status step.
   *
   * @param {MfwAction} action Updated action.
   */

  /**
   * @ngdoc service
   * @name mfw.business.actionstore.service:$mfwActions
   *
   * @requires flux
   * @requires mfw.business.actionstore.service:$mfwActionStore
   *
   * @description
   * The `$mfwActions` service acts as an action creator for all your promises and dispatches processes
   * to {@link mfw.business.actionstore.service:$mfwActionStore `$mfwActionStore`}.
   */
  ActionStoreModule.service('$mfwActions', ActionCreator);
  ActionCreator.$inject = ['$timeout', 'flux', '$mfwActionStore'];
  function ActionCreator($timeout, flux, $mfwActionStore) {
    /**
     * @ngdoc method
     * @name mfw.business.actionstore.service:$mfwActions#newAction
     * @methodOf mfw.business.actionstore.service:$mfwActions
     *
     * @description
     * This method takes a promise and handles its lifecycle broadcasting
     * {@link mfw.business.actionstore.service:$mfwActionStore#events store events}.
     *
     * It dispatches the `$mfwActionCreation` action handled by {@link mfw.business.actionstore.service:$mfwActionStore `$mfwActionStore`}.
     *
     * @param {Function|Promise} actionBody The promise or a function that returns a promise.
     * @returns {Promise} A wrapped promise with the same resolution/rejection process than the original one.
     */
    this.newAction = function (actionBody) {
      var actionId = _newActionId();
      var originalPromise = angular.isFunction(actionBody) ? actionBody() : actionBody;
      var promise = _wrapPromise(originalPromise, actionId);

      // Avoid more than one flux dispatch at the same time
      $timeout(function () {
        flux.dispatch('$mfwActionCreation', promise);
      });

      return promise;
    };

    //////////////////////////

    /**
     * @description
     * This method wraps the action promise with an internal `$mfwId` identifier.
     *
     * It also dispatches events `$mfwActionFinished` and `$mfwActionError` whenever the
     * promise is resolved or rejected.
     *
     * @param {Promise} promise Action promise.
     * @param {String} actionId New action identifier.
     * @returns {Promise} - Incomming promise wrapped with `$mfwId` identifier.
     * @private
     */
    function _wrapPromise(promise, actionId) {
      var result = promise.then(resolveHandler, rejectHandler);
      result.$mfwId = actionId;
      return result;

      /////////////////////

      /**
       * @description
       * This method is executed when the action promise is resolved, and its main purpose
       * is fire the `$mfwActionFinished` flux action.
       *
       * @returns {*} The original action promise results, if any.
       * @private
       */
      function resolveHandler() {
        flux.dispatch('$mfwActionFinished', actionId);
        return (arguments || []).length ? arguments[0] : undefined;
      }

      /**
       * @description
       * This method is executed when the action promise is rejected, and its main purpose
       * is fire the `$mfwActionError` flux action.
       *
       * @returns {*} The original action promise results, if any.
       * @private
       */
      function rejectHandler() {
        flux.dispatch('$mfwActionError', actionId);
        return (arguments || []).length ? arguments[0] : undefined;
      }
    }

    /**
     * @description
     * This method generates a new random ID to be used as action identifier.
     *
     * @returns {string} New action identifier.
     * @private
     */
    function _newActionId() {
      function s4() {
        return Math.floor((1 + Math.random()) * 0x10000)
          .toString(16)
          .substring(1);
      }

      return s4() + s4() + '-' + s4() + '-' + s4() + '-' +
        s4() + '-' + s4() + s4() + s4();
    }
  }

  /**
   * @ngdoc service
   * @name mfw.business.actionstore.service:$mfwActionStore
   *
   * @requires $log
   *
   * @description
   * This service takes care of all promises lifecycle and offers an API to ask for their status,
   * as well as notifying status changes via flux events:
   *
   * * {@link mfw.business.actionstore.service:$mfwActionStore#events `mfw:Action.<actionId>.<status>`}: events by action, useful to take care of an specific action status changes.
   * * {@link mfw.business.actionstore.service:$mfwActionStore#events `mfw:Action.<status>.<actionId>`}: events by status, useful for loggers and error handlers.
   *
   * Were `<actionId>` should be replaced by the action identifier associated to the created action. See
   * {@link mfw.business.actionstore.service:$mfwActionStore#methods_actionId `$mfwActionStore.actionId()`} for more information.
   *
   * And `<status>` takes one of the following values:
   *
   * * `running`: action starts running
   * * `finished`: action finished successfully
   * * `error`: action finished with error (with additional error information)
   */
  ActionStoreModule.store('$mfwActionStore', ActionStore);
  ActionStore.$inject = ['$log'];
  function ActionStore($log) {
    var store = {
      initialize: function initialize() {
        var initialState = {
          actions: {}
        };
        this.state = initialState;
      },
      handlers: {
        $mfwActionCreation: 'newAction',
        $mfwActionFinished: 'actionFinished',
        $mfwActionError: 'actionError'
      },
      /**
       * @description
       * Handler for the `$mfwActionCreation` flux action.
       *
       * This method takes a wrapped promise and creates the associated `MfwAction` action.
       *
       * By default, newly created actions have `STATUS_RUNNING` status value.
       *
       * @param {Promise} promise The promise to be wrapped.
       * @returns {String} The identifier of the new action.
       * @private
       */
      newAction: function newAction(promise) {
        var newAction = {
          id: promise.$mfwId,
          promise: promise,
          status: STATUS_RUNNING
        };
        this.state.actions[newAction.id] = newAction;

        $log.debug('> New action', newAction.id);

        this.broadcastActionUpdate(newAction);
        return newAction.id;
      },
      /**
       * @description
       * Handler for the `$mfwActionFinished` flux action.
       *
       * This method updates the associated action with the `STATUS_FINISHED` status value.
       *
       * @param {String} actionId The action identifier.
       */
      actionFinished: function actionFinished(actionId) {
        $log.debug('> Finished action', actionId);
        this.setActionStatus(actionId, STATUS_FINISHED);
      },
      /**
       * @description
       * Handler for the `$mfwActionError` flux action.
       *
       * This method updates the associated action with the `STATUS_ERROR` status value.
       *
       * @param {String} actionId The action identifier.
       * @param {*} error Error information.
       */
      actionError: function actionError(actionId, error) {
        $log.debug('> Error in action', actionId);
        this.setActionStatus(actionId, STATUS_ERROR, error);
      },
      /**
       * @description
       * Helper method to get an action by its identifier.
       *
       * @param {String} actionId The action identifier.
       * @returns {MfwAction} The action.
       */
      getAction: function getAction(actionId) {
        $log.debug('> Get action', actionId);
        return this.state.actions[actionId];
      },
      /**
       * @description
       * Helper method that emits flux events according to the status of the given action.
       *
       * It emits events for action handlers and for status handlers.
       *
       * @param {MfwAction} action The action.
       */
      broadcastActionUpdate: function broadcastActionUpdate(action) {
        var actionId = action.id;
        var status = action.status;
        $log.debug('> Broadcast action update', actionId, 'with status', status);

        // Different events
        this.emit(actionEvent(action, status), action);
        this.emit(statusEvent(action, status), action);
      },
      /**
       * @description
       * Helper method that updates an action's status and triggers the associated events.
       *
       * It emits events for action handlers and for status handlers.
       *
       * @param {String} actionId The action identifier.
       * @param {String} status New status.
       * @param {*=} data Additional information, like an error.
       */
      setActionStatus: function setActionStatus(actionId, status, data) {
        $log.debug('> Update action', actionId, 'with status', status, 'and data', data);
        angular.extend(this.state.actions[actionId], {
          status: status,
          data: data
        });
        this.broadcastActionUpdate(this.state.actions[actionId]);
      },

      exports: {
        /**
         * @ngdoc method
         * @name mfw.business.actionstore.service:$mfwActionStore#getAction
         * @methodOf mfw.business.actionstore.service:$mfwActionStore
         *
         * @description
         * This method returns the associated `MfwAction` instance associated to the action promise.
         *
         * @param {Promise} promise The promise wrapping the action.
         * @returns {MfwAction} Specified action.
         * @private
         */
        getAction: getAction,
        /**
         * @ngdoc method
         * @name mfw.business.actionstore.service:$mfwActionStore#actionId
         * @methodOf mfw.business.actionstore.service:$mfwActionStore
         *
         * @description
         * This method returns the action identifier associated to the action promise.
         *
         * @param {Promise} promise The promise wrapping the action.
         * @returns {String} Action identifier.
         * @private
         */
        actionId: actionId,
        /**
         * @ngdoc method
         * @name mfw.business.actionstore.service:$mfwActionStore#allEvents
         * @methodOf mfw.business.actionstore.service:$mfwActionStore
         *
         * @description
         * This method returns the event identifier to be used in `$scope.$listenTo` method
         * to be noticed for all action events.
         *
         * If a promise is specified, will return the *all event's* key for that action, otherwise the identifier
         * for all action events.
         *
         * Returned event identifiers:
         *
         * * `mfw:Action.*.*`: all events of all actions and status.
         * * `mfw:Action.<actionId>.*`: all events of a specific action.
         *
         * @param {Promise=} promise The promise wrapping the action, if any.
         * @returns {String} Event identifier.
         * @private
         */
        allEvents: function (promise) {
          return promise ?
            allEvents(this.getAction(actionId(promise)))
            : allEvents();
        },
        /**
         * @ngdoc method
         * @name mfw.business.actionstore.service:$mfwActionStore#finishEvent
         * @methodOf mfw.business.actionstore.service:$mfwActionStore
         *
         * @description
         * This method returns the event identifier to be used in `$scope.$listenTo` method
         * to be noticed for finish action event.
         *
         * If a promise is specified, will return the error event for that action, otherwise the identifier
         * for all actions' error event.
         *
         * Returned event identifiers:
         *
         * * `mfw:Action.finished.*`: all finished events of all actions and status.
         * * `mfw:Action.<actionId>.finished`: finished event of a specific action.
         *
         * @param {Promise=} promise The promise wrapping the action, if any.
         * @returns {String} Event identifier.
         * @private
         */
        finishEvent: function (promise) {
          return promise ?
            finishEvent(this.getAction(actionId(promise)))
            : finishEvent();
        },
        /**
         * @ngdoc method
         * @name mfw.business.actionstore.service:$mfwActionStore#errorEvent
         * @methodOf mfw.business.actionstore.service:$mfwActionStore
         *
         * @description
         * This method returns the event identifier to be used in `$scope.$listenTo` method
         * to be noticed for error action event.
         *
         * If a promise is specified, will return the error event for that action, otherwise the identifier
         * for all actions' error event.
         *
         * Returned event identifiers:
         *
         * * `mfw:Action.error.*`: all error events of all actions and status.
         * * `mfw:Action.<actionId>.error`: error event of a specific action.
         *
         * @param {Promise} promise The promise wrapping the action.
         * @returns {String} Event identifier.
         * @private
         */
        errorEvent: function (promise) {
          return promise ?
            errorEvent(this.getAction(actionId(promise)))
            : errorEvent();
        }
      }
    };

    /**
     * @description
     * This method returns the associated action id, in `$mfwId` property.
     *
     * @param {Promise} promise Wrapped action promise.
     * @returns {String} Action identifier.
     * @private
     */
    function actionId(promise) {
      return promise.$mfwId;
    }

    /**
     * @description
     * This method returns the event identifier to be used with `$scope.$listenTo` method.
     *
     * If an action is specified, will return the finish event for that action, otherwise the identifier
     * for all actions' finish event.
     *
     * @param {MfwAction=} action Action, if any.
     * @returns {String} Event identifier.
     * @private
     *
     * @see actionEvent
     * @see statusEvent
     */
    function finishEvent(action) {
      return action ?
        actionEvent(action, STATUS_FINISHED)
        : statusEvent(null, STATUS_FINISHED);
    }

    /**
     * @description
     * This method returns the event identifier to be used with `$scope.$listenTo` method.
     *
     * If an action is specified, will return the error event for that action, otherwise the identifier
     * for all actions' error event.
     *
     * @param {MfwAction=} action Action, if any.
     * @returns {String} Event identifier.
     * @private
     *
     * @see actionEvent
     * @see statusEvent
     */
    function errorEvent(action) {
      return action ?
        actionEvent(action, STATUS_ERROR)
        : statusEvent(null, STATUS_ERROR);
    }

    /**
     * @description
     * This method returns the event identifier to be used with `$scope.$listenTo` method.
     *
     * If an action is specified, will return the *all event's* key for that action, otherwise the identifier
     * for all action events.
     *
     * @param {MfwAction=} action Action, if any.
     * @returns {String} Event identifier.
     * @private
     */
    function allEvents(action) {
      return ACTION_PREFIX
        + (action ? action.id + '.*' : '*');
    }

    /**
     * @description
     * This method returns the event identifier to be used with `$scope.$listenTo` method.
     *
     * If an action is specified, will return the finish event for that action, otherwise the identifier
     * for all actions' finish event.
     *
     * @param {MfwAction=} action Action, if any.
     * @returns {String} Event identifier.
     * @private
     */
    function actionEvent(action, status) {
      return ACTION_PREFIX
        + (action ? action.id : '*')
        + '.'
        + status;
    }

    /**
     * @description
     * This method returns the event identifier to be used with `$scope.$listenTo` method.
     *
     * If an action is specified, will return the finish event for that action, otherwise the identifier
     * for all actions' finish event.
     *
     * @param {MfwAction=} action Action, if any.
     * @returns {String} Event identifier.
     * @private
     */
    function statusEvent(action, status) {
      return ACTION_PREFIX
        + status
        + '.'
        + (action ? action.id : '*');
    }

    function getAction(promise) {
      return store.getAction(actionId(promise));
    }

    return store;
  }
})();


text:
android-storage
===============

Android Storage done right following the recommended approach described in the documentation for [internal storage](http://developer.android.com/guide/topics/data/data-storage.html#filesInternal) and [external storage](http://developer.android.com/guide/topics/data/data-storage.html#filesExternal)