text:
package org.ovirt.engine.core.bll;

import java.util.ArrayList;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.locks.Lock;
import java.util.concurrent.locks.ReentrantLock;

import org.ovirt.engine.core.common.AuditLogType;
import org.ovirt.engine.core.common.businessentities.FileTypeExtension;
import org.ovirt.engine.core.common.businessentities.RepoFileMetaData;
import org.ovirt.engine.core.common.businessentities.StorageDomainStatus;
import org.ovirt.engine.core.common.businessentities.StorageDomainType;
import org.ovirt.engine.core.common.businessentities.StoragePoolStatus;
import org.ovirt.engine.core.common.businessentities.VDSStatus;
import org.ovirt.engine.core.common.businessentities.storage_domains;
import org.ovirt.engine.core.common.businessentities.storage_pool_iso_map;
import org.ovirt.engine.core.common.config.Config;
import org.ovirt.engine.core.common.config.ConfigValues;
import org.ovirt.engine.core.common.vdscommands.IrsBaseVDSCommandParameters;
import org.ovirt.engine.core.common.vdscommands.VDSCommandType;
import org.ovirt.engine.core.common.vdscommands.VDSReturnValue;
import org.ovirt.engine.core.compat.Guid;
import org.ovirt.engine.core.compat.LogCompat;
import org.ovirt.engine.core.compat.LogFactoryCompat;
import org.ovirt.engine.core.compat.TransactionScopeOption;
import org.ovirt.engine.core.dal.dbbroker.DbFacade;
import org.ovirt.engine.core.dal.dbbroker.auditloghandling.AuditLogDirector;
import org.ovirt.engine.core.dal.dbbroker.auditloghandling.AuditLogableBase;
import org.ovirt.engine.core.dao.RepoFileMetaDataDAO;
import org.ovirt.engine.core.utils.Pair;
import org.ovirt.engine.core.utils.threadpool.ThreadPoolUtil;
import org.ovirt.engine.core.utils.timer.OnTimerMethodAnnotation;
import org.ovirt.engine.core.utils.timer.SchedulerUtilQuartzImpl;
import org.ovirt.engine.core.utils.transaction.TransactionMethod;
import org.ovirt.engine.core.utils.transaction.TransactionSupport;

/**
 * The class manages the Iso domain cache mechanism,

 * which reflects upon support for Iso tool validation, activation of Iso domain, and fetching the Iso list by query.

 * The cache is being refreshed with quartz scheduler which run by configuration value AutoRepoDomainRefreshTime. The
 * cache procedure using VDSM to fetch the Iso files from all the Data Centers and update the DB cache table with the
 * appropriate file data.

 */
public class IsoDomainListSyncronizer {
    private List<RepoFileMetaData> problematicRepoFileList = new ArrayList<RepoFileMetaData>();
    public static final String guestToolsSetupIsoPrefix = Config.<String> GetValue(ConfigValues.GuestToolsSetupIsoPrefix);
    public static final String regexToolPattern = guestToolsSetupIsoPrefix + "([0-9]{1,}.[0-9])(_{1})([0-9]{1,}).[i|I][s|S][o|O]$";
    private final int MIN_TO_MILLISECONDS = 60 * 1000;
    private static final IsoDomainListSyncronizer isoDomainListSyncronizer = new IsoDomainListSyncronizer();;
    private int isoDomainRefreshRate;
    RepoFileMetaDataDAO repoStorageDom;
    private static ConcurrentMap<Object,Lock> syncDomainForFileTypeMap = new ConcurrentHashMap<Object,Lock>();

    /**
     * private constructor to initialize the quartz scheduler
     */
    private IsoDomainListSyncronizer() {
        repoStorageDom = DbFacade.getInstance().getRepoFileMetaDataDao();
        isoDomainRefreshRate = Config.<Integer> GetValue(ConfigValues.AutoRepoDomainRefreshTime) * MIN_TO_MILLISECONDS;
        SchedulerUtilQuartzImpl.getInstance().scheduleAFixedDelayJob(this,
                "fetchIsoDomains",
                new Class[] {},
                    new Object[] {},
                300000,
                isoDomainRefreshRate,
                TimeUnit.MILLISECONDS);
    }

    /**
     * Returns the singleton instance.
     * @return Singleton instance of IsoDomainManager
     */
    public static IsoDomainListSyncronizer getInstance() {
        return isoDomainListSyncronizer;
    }

    /**
     * Check and update if needed each Iso domain in each Data Center in the system.
     */
    @OnTimerMethodAnnotation("fetchIsoDomains")
    public void fetchIsoDomains() {
        // Gets all the active Iso storage domains.
        List<RepoFileMetaData> repofileList = DbFacade.getInstance()
                        .getRepoFileMetaDataDao()
                        .getAllRepoFilesForAllStoragePools(StorageDomainType.ISO,
                                StoragePoolStatus.Up,
                                StorageDomainStatus.Active,
                                VDSStatus.Up);

        // Set count down latch to size of map for multiple threaded.
        final CountDownLatch latch = new CountDownLatch(repofileList.size());

        resetProblematicList();
        // Iterate for each storage domain.
        for (final RepoFileMetaData repoFileMetaData : repofileList) {
            // If the list should be refreshed and the refresh from the VDSM was succeeded, fetch the file list again
            // from the DB.
            if (shouldRefreshIsoDomain(repoFileMetaData.getLastRefreshed())) {
                ThreadPoolUtil.execute(new Runnable() {
                    @Override
                    public void run() {
                        updateCachedIsoFileListFromVdsm(repoFileMetaData, latch);
                    }
                });
            } else {
                latch.countDown();
                log.debugFormat("Automatic refresh process for {0} file type in storage domain id {1} was not performed since refresh time out did not passed yet.",
                        repoFileMetaData.getFileType(),
                        repoFileMetaData.getRepoDomainId());
            }
        }

        try {
            latch.await();
        } catch (InterruptedException e) {
            log.error("Automatic refresh process encounter a problem.", e);
        }

        // After refresh for all Iso domains finished, handle the log.
        handleErrorLog(problematicRepoFileList);
    }

    /**
     * Returns a RepoFilesMetaData list with Iso file names for storage domain Id and with file type extension.

     * If user choose to refresh the cache, and a problem occurs, then returns null.
     *
     * @param storageDomainId
     *            - The storage domain Id, which we fetch the Iso list from.
     * @param fileTypeExtension
     *            - The fileTypeExtension we want to fetch the files from the cache.
     * @param forceRefresh
     *            - Indicates if the domain should be refreshed from VDSM.
     * @return List of RepoFilesMetaData files or null (If fetch from VDSM failed).
     */
    public List<RepoFileMetaData> getUserRequestForStorageDomainRepoFileList(Guid storageDomainId,
            FileTypeExtension fileTypeExtension,
            boolean forceRefresh) {
        // The result list we send back.
        List<RepoFileMetaData> repoList = null;

        // Check storage pool Id validity.
        if (storageDomainId == null) {
            log.error("Storage domain ID recieved from command query is null.");
            return null;
        }

        List<RepoFileMetaData> tempProblematicRepoFileList = new ArrayList<RepoFileMetaData>();

        // If user choose to force refresh.
        if (forceRefresh) {
            // Add an audit log if refresh succeeded.
            if (refreshIsoDomain(storageDomainId, tempProblematicRepoFileList, fileTypeExtension)) {
                // Print log Indicating the problematic pools (If was any).
                handleErrorLog(tempProblematicRepoFileList);

                // If refresh succeeded print an audit log Indicating that.
                storage_domains storageDomain = DbFacade.getInstance().getStorageDomainDAO().get(storageDomainId);
                addToAuditLogSuccessMessage(storageDomain.getstorage_name(), fileTypeExtension.name());
            } else {
                // Print log Indicating the problematic pools.
                handleErrorLog(tempProblematicRepoFileList);
                return null;
            }
        }

        // At any case, if refreshed or not, get Iso list from the cache.
        repoList = getCachedIsoListByDomainId(storageDomainId, fileTypeExtension);

        // Return list of repository files.
        return repoList;
    }

    /**
     * The procedure Try to refresh the repository files of the storage domain id, with storage pool Id. If succeeded
     * will return True, otherwise return false and update the list, of the problematic repository files with the
     * storage pool and storage domain id, that could not complete the cache update transaction.
     *
     * @param storageDomainId
     *            - The Repository domain Id, we want to refresh.
     * @param storagePoolId
     *            - The Storage pool Id, we use to fetch the Iso files from.
     * @param ProblematicRepoFileList
     *            - List of business entities, each one indicating the problematic entity of storage pool id, storage
     *            domain id, and file type.
     * @param fileTypeExtension
     *            - The fileTypeExtension we want to fetch the files from the cache.
     * @return Boolean value indicating if the refresh succeeded or not.
     */
    private boolean refreshIsoDomainFileForStoragePool(Guid storageDomainId,
            Guid storagePoolId,
            FileTypeExtension fileTypeExt) {
        boolean refreshSucceeded = false;

        // Check validation of storage pool and SPM.
        Boolean isValid = (Boolean) (Backend
                .getInstance()
                .getResourceManager()
                .RunVdsCommand(VDSCommandType.IsValid,
                        new IrsBaseVDSCommandParameters(storagePoolId)).getReturnValue());

        log.debugFormat("Validation of Storage pool id {0} returned {1}.", storagePoolId, isValid ? "valid" : "not valid");
        // Setting the indication to the indication whether the storage pool is valid.
        boolean updateFromVDSMSucceeded = isValid;

        // If the SPM and the storage pool are valid, try to refresh the Iso list by fetching it from the SPM.
        if (isValid) {
            if (fileTypeExt == FileTypeExtension.ISO) {
                updateFromVDSMSucceeded = updateIsoListFromVDSM(storagePoolId, storageDomainId);
            } else if (fileTypeExt == FileTypeExtension.Floppy) {
                updateFromVDSMSucceeded =
                        updateFloppyListFromVDSM(storagePoolId, storageDomainId) && updateFromVDSMSucceeded;
            }
        }

        // Log if the refresh succeeded or add the storage domain to the problematic list.
        if (updateFromVDSMSucceeded) {
            refreshSucceeded = true;
            log.debugFormat("Refresh succeeded for file type {0} at storage domain id {1} in storage pool id {2}.",
                    fileTypeExt.name(),
                    storageDomainId,
                    storagePoolId);
        }
        return refreshSucceeded;
    }

    /**
     * The procedure Try to refresh the repository files of the storage domain id, By iterate over the storage pools of
     * this domain, and try to choose a valid storage pool, to fetch the repository files from the VDSM, and refresh the
     * cached table.

     * If succeeded, will return True. Otherwise return false with updated list of problematic repository files with the
     * storage pool, storage domain, and file type, that could not complete the cache update transaction.
     *
     * @param storageDomainId
     *            - The Repository domain Id, we want to refresh.
     * @param ProblematicRepoFileList
     *            - List of business entities, each one indicating the problematic entity.
     * @param fileTypeExtension
     *            - The fileTypeExtension we want to fetch the files from the cache.
     * @return Boolean value indicating if the refresh succeeded or not.
     */
    private boolean refreshIsoDomain(Guid storageDomainId,
            List<RepoFileMetaData> problematicRepoFileList,
            FileTypeExtension fileTypeExt) {
        boolean refreshSucceeded = false;
        List<RepoFileMetaData> tempProblematicRepoFileList = new ArrayList<RepoFileMetaData>();

        // Fetch all the Storage pools for this Iso domain Id.
        List<storage_pool_iso_map> isoMapList =
                DbFacade.getInstance()
                        .getStoragePoolIsoMapDAO()
                        .getAllForStorage(storageDomainId);
        log.debugFormat("Fetched {0} storage pools for {1} file type, in Iso domain {2}.",
                isoMapList.size(),
                fileTypeExt,
                storageDomainId);
        Iterator<storage_pool_iso_map> iter = isoMapList.iterator();

        while (iter.hasNext() && !refreshSucceeded) {
            storage_pool_iso_map storagePoolIsoMap = iter.next();
            Guid storagePoolId = storagePoolIsoMap.getstorage_pool_id().getValue();
            if ((storagePoolIsoMap.getstatus() != null)
                    && (storagePoolIsoMap.getstatus() == StorageDomainStatus.Active)) {
                // Try to refresh the domain of the storage pool id.
                refreshSucceeded =
                        refreshIsoDomainFileForStoragePool(storageDomainId,
                                storagePoolId,
                                fileTypeExt);
            } else {
                log.debugFormat("Storage domain id {0}, is not active, and there for could not be refreshed for {1} file type (Iso domain status is {2}).",
                        storageDomainId,
                        fileTypeExt,
                        storagePoolIsoMap.getstatus());
            }

            if (!refreshSucceeded) {
                log.debugFormat("Failed refreshing Storage domain id {0}, for {1} file type in storage pool id {2}.",
                        storageDomainId,
                        fileTypeExt,
                        storagePoolId);

                // set a mock repository file meta data with storage domain id and storage pool id.
                RepoFileMetaData repoFileMetaData = new RepoFileMetaData();
                repoFileMetaData.setStoragePoolId(storagePoolId);
                repoFileMetaData.setRepoDomainId(storageDomainId);
                repoFileMetaData.setFileType(fileTypeExt);

                // Add the repository file to the list of problematic Iso domains.
                tempProblematicRepoFileList.add(repoFileMetaData);
            }
        }

        // If refreshed was not succeeded add the problematic storage Iso domain to the list.
        if (!refreshSucceeded) {
            problematicRepoFileList.addAll(tempProblematicRepoFileList);
        }

        return refreshSucceeded;
    }

    /**
     * Refresh the Iso domain when activating the domain,
     * with executing a new Thread to prevent long lock status for the domain.
     *
     * @param IsoStorageDomainId
     *            - The storage domain Id we want to get the file list from.
     * @param storagePoolId
     *            - The storage pool Id we get an Iso active domain, we want to get the file list from (used mainly for log issues).
     */
    public void refresheIsoDomainWhenActivateDomain(final Guid isoStorageDomainId,
            final Guid storagePoolId) {
        if (storagePoolId != null && (isoStorageDomainId != null))
        {
            ThreadPoolUtil.execute(new Runnable() {
                @Override
                public void run() {
                    refreshActivatedStorageDomainFromVdsm(storagePoolId, isoStorageDomainId);
                }
            });
        }
    }

    /**
     * Returns the cached Iso file meta data list, of the storage pool Id with the storage domain id.
     *
     * @param IsoStorageDomainId
     *            - The storage domain Id we want to get the file list from.
     * @param IsoStoragePoolId
     *            - The storage pool Id we want to get the file list from.
     * @return List of Iso file fetched from DB, if parameter is invalid returns an empty list.
     */
    public List<RepoFileMetaData> getCachedIsoListByPoolAndDomainId(Guid isoStoragePoolId, Guid isoStorageDomainId) {
        List<RepoFileMetaData> fileListMD = new ArrayList<RepoFileMetaData>();
        // Check validation of parameters.
        if (isoStorageDomainId != null && isoStoragePoolId != null) {
            // Get all the Iso files of storage and domain ID.
            fileListMD = repoStorageDom.getRepoListForStorageDomainAndStoragePool(isoStoragePoolId, isoStorageDomainId,
                    FileTypeExtension.ISO);
        }
        return fileListMD;
    }

    /**
     * Returns the cached Iso file meta data list, for storage domain.
     *
     * @param IsoStorageDomainId
     *            - The storage domain Id we want to get the file list from.
     * @return List of Iso files fetched from DB, if parameter is invalid returns an empty list.
     */
    public List<RepoFileMetaData> getCachedIsoListByDomainId(Guid isoStorageDomainId,FileTypeExtension fileTypeExtension) {
        List<RepoFileMetaData> fileListMD = new ArrayList<RepoFileMetaData>();
        if (isoStorageDomainId != null) {
            fileListMD =
                    repoStorageDom.getRepoListForStorageDomain(isoStorageDomainId, fileTypeExtension);
        }
        return fileListMD;
    }

    /**
     * Handling the list of problematic repository files, to maintain multi thread caching.
     * @see #resetProblematicList()
     */
    private synchronized void addRepoFileToProblematicList(List<RepoFileMetaData> repoFileMetaDataList) {
        problematicRepoFileList.addAll(repoFileMetaDataList);
    }

    /**
     * Reset the list of problematic repository files, before starting the refresh procedure.
     * uses for multy thread caching.
     * @see #addRepoFileToProblematicList()
     */
    private synchronized void resetProblematicList() {
        problematicRepoFileList.clear();
    }

    /**
     * Print information on the problematic storage domain. Mainly transfer the business entity to list, for handling
     * the error uniformly.
     * Create a mock RepoFileMetaData object in a list, to use the functionality of the handleErrorLog with list.
     *
     * @param IsoStorageDomainId
     *            - The storage domain Id.
     * @param storagePoolId
     *            - The storage pool Id.
     * @param fileTypeExt
     *            - The file type extension (ISO  or Floppy).
     * @see #handleErrorLog(List)
     */
    private void handleErrorLog(Guid storagePoolId, Guid storageDomainId, FileTypeExtension fileTypeExt) {
        List<RepoFileMetaData> tempProblematicRepoFileList = new ArrayList<RepoFileMetaData>();

        // set mock repo file meta data with storage domain id and storage pool id.
        RepoFileMetaData repoFileMetaData = new RepoFileMetaData();
        repoFileMetaData.setStoragePoolId(storagePoolId);
        repoFileMetaData.setRepoDomainId(storageDomainId);
        repoFileMetaData.setFileType(fileTypeExt);

        // Add the repository file to the list, and use handleError.
        tempProblematicRepoFileList.add(repoFileMetaData);
        handleErrorLog(tempProblematicRepoFileList);
    }

    /**
     * Print information on the problematic storage domains and print an audit log.

     * If the problematicFileListForHandleError list retrieved empty or null,

     * then don't do nothing and return false flag.
     *
     * @param problematicFileListForHandleError
     *            - List of repository file meta data, each one indicating a problematic repository domain.
     * @return true, if has problematic storage domains, false otherwise (List is empty).
     */
    private boolean handleErrorLog(List<RepoFileMetaData> problematicFileListForHandleError) {
        boolean hasProblematic = false;
        if (problematicFileListForHandleError != null && !problematicFileListForHandleError.isEmpty()) {
            StringBuilder problematicStorages = new StringBuilder();
            StringBuilder problematicIsoDomainsForAuditLog = new StringBuilder();
            Set<String> storageDomainNames = new HashSet<String>();
            for (RepoFileMetaData repoMap : problematicFileListForHandleError) {
                problematicStorages.append(buildDetailedProblematicMapMsg(repoMap));
                storageDomainNames.add(buildDetailedAuditLogMessage(repoMap));
            }

            // Build Audit log message with problematic domains.
            for (String domainName : storageDomainNames) {
                problematicIsoDomainsForAuditLog.append("  ").append(domainName);
            }

            hasProblematic = true;
            log.errorFormat("The following storage domains had a problem retrieving data from VDSM {0}",
                    problematicStorages.toString());
            addToAuditLogErrorMessage(problematicIsoDomainsForAuditLog.toString());
        }
        return hasProblematic;
    }

    /**
     * Returns a string builder contains problematic repoFileMetaData details.
     *
     * @param repoFileMetaData
     *            - The problematic storage domain.
     */
    private StringBuilder buildDetailedProblematicMapMsg(RepoFileMetaData repoFileMetaData) {
        StringBuilder problematicStorageMsg = new StringBuilder();
        if (repoFileMetaData != null) {
            problematicStorageMsg.append(" (");
            if (repoFileMetaData.getStoragePoolId() != null) {
                problematicStorageMsg.append(" Storage Pool Id: ").append(repoFileMetaData.getStoragePoolId());
            }
            if (repoFileMetaData.getRepoDomainId() != null) {
                problematicStorageMsg.append(" Storage domain Id: ").append(repoFileMetaData.getRepoDomainId());
            }
            problematicStorageMsg.append(" File type: ").append(repoFileMetaData.getFileType()).append(") ");
        } else {
            problematicStorageMsg.append("(A repository file meta data business entity, has null value) ");
        }
        return problematicStorageMsg;
    }

    /**
     * Returns String contains problematic iso domain name for audit log message.
     * @param repoFileMetaData
     *                  - The problematic storage domain.
     * @return
     */
    private String buildDetailedAuditLogMessage(RepoFileMetaData repoFileMetaData) {
        String storageDomainName = "Repository not found";
        if (repoFileMetaData != null && repoFileMetaData.getRepoDomainId() != null) {
            storage_domains storageDomain = DbFacade.getInstance().getStorageDomainDAO().get(repoFileMetaData.getRepoDomainId());
            if (storageDomain != null) {
                storageDomainName =
                        String.format("%s (%s file type)",
                                storageDomain.getstorage_name(),
                                repoFileMetaData.getFileType().name());
            }
        } else {
            log.error("Repository file meta data not found for logging");
        }
        return storageDomainName;
    }

    /**
     * Updates the DB cache table with files fetched from VDSM.
     * The method is dedicated for multiple threads refresh.
     * If refresh from VDSM has encounter problems, we update the problematic domain list.
     * @param repoFileMetaData
     */
    private void updateCachedIsoFileListFromVdsm(RepoFileMetaData repoFileMetaData, final CountDownLatch latch)
    {
        boolean isRefreshed = false;
        try {
            List<RepoFileMetaData> problematicRepoFileList = new ArrayList<RepoFileMetaData>();
            isRefreshed = refreshIsoDomain(repoFileMetaData.getRepoDomainId(),problematicRepoFileList,repoFileMetaData.getFileType());
            addRepoFileToProblematicList(problematicRepoFileList);
        } finally {
            // At any case count down the latch, and print log message.
            latch.countDown();
            log.infoFormat("Finished automatic refresh process for {0} file type with {1}, for storage domain id {2}.",
                    repoFileMetaData.getFileType(),
                    isRefreshed ? "success"
                            : "failure",
                    repoFileMetaData.getRepoDomainId());
        }
    }

    /**
     * Create a new transaction to refresh the Iso file list in to the DB.
     *
     * @param repoDomainId
     *            - The repository domain id we want to update.
     * @param storageDomainDao
     *            - The Data Access Layer for storage domain.
     * @param isoDomainList
     *            - The Iso files which refreshed in the cache.
     */
    private static boolean refreshIsoFileListMetaData(final Guid repoStorageDomainId,
            final RepoFileMetaDataDAO repoFileMetaDataDao,
            final List<String> isoDomainList, final FileTypeExtension fileTypeExt) {
        Lock syncObject = getSyncObject(repoStorageDomainId, fileTypeExt);
        try {
            syncObject.lock();
            return (Boolean) TransactionSupport.executeInScope(TransactionScopeOption.RequiresNew,
                    new TransactionMethod<Object>() {
                        @Override
                        public Object runInTransaction() {
                            long currentTime = System.currentTimeMillis();
                            repoFileMetaDataDao.removeRepoDomainFileList(repoStorageDomainId, fileTypeExt);
                            RepoFileMetaData repo_md;
                            for (String isoFile : isoDomainList) {
                                repo_md = new RepoFileMetaData();
                                repo_md.setLastRefreshed(currentTime);
                                repo_md.setSize(0);
                                repo_md.setRepoDomainId(repoStorageDomainId);
                                repo_md.setDateCreated(null);
                                repo_md.setRepoFileName(isoFile);
                                repo_md.setFileType(fileTypeExt);
                                repoFileMetaDataDao.addRepoFileMap(repo_md);
                            }
                            return true;
                        }
                    });
        } finally {
            syncObject.unlock();
        }
    }

    /**
     * Try to update the cached table from the VDSM, if succeeded fetch the file list again from the DB. if not ,handle
     * the log message.
     *
     * @param IsoStorageDomainId
     *            - The storage domain Id we want to get the file list from.
     * @param fileListMD
     *            - File list of repository files.
     * @param repoFileMetaData
     *            - repository file we fetch the domain counting on its domain id.
     */
    private synchronized void refreshActivatedStorageDomainFromVdsm(Guid storagePoolId, Guid storageDomainId) {
        if (!updateIsoListFromVDSM(storagePoolId, storageDomainId)) {
            // Add an audit log that refresh was failed for Iso files.
            handleErrorLog(storagePoolId, storageDomainId, FileTypeExtension.ISO);
        }
        if (!updateFloppyListFromVDSM(storagePoolId, storageDomainId)) {
            // Add an audit log that refresh was failed for Floppy files.
            handleErrorLog(storagePoolId, storageDomainId, FileTypeExtension.Floppy);
        }
    }

    /**
     * Check if last refreshed time has exceeded the time limit configured in isoDomainRefreshRate.
     *
     * @param lastRefreshed
     *            - Time when repository file was last refreshed.
     * @return True if time exceeded, and should refresh the domain, false otherwise.
     */
    private boolean shouldRefreshIsoDomain(long lastRefreshed) {
        return ((System.currentTimeMillis() - lastRefreshed) > isoDomainRefreshRate);
    }

    /**
     * Gets the Iso file list from VDSM, and if the fetch is valid refresh the Iso list in the DB.
     *
     * @param repoFileMetaData
     *            - The repository storage pool id, we want to update the file list.
     * @param repoStorageDomainId
     *            - The repository storage domain id, for activate storage domain id.
     * @return True, if the fetch from VDSM has succeeded. False otherwise.
     */
    private boolean updateIsoListFromVDSM(Guid repoStoragePoolId, Guid repoStorageDomainId) {
        boolean refreshIsoSucceeded = false;
        if (repoStorageDomainId != null) {
            try {
                // Get Iso domain file list from storage pool.
                VDSReturnValue returnValue = Backend
                        .getInstance()
                        .getResourceManager()
                        .RunVdsCommand(VDSCommandType.GetIsoList,
                                new IrsBaseVDSCommandParameters(repoStoragePoolId));
                @SuppressWarnings("unchecked")
                List<String> isoDomainList = (List<String>) returnValue.getReturnValue();
                if (returnValue.getSucceeded() && isoDomainList != null) {
                    log.debugFormat("The refresh process from VDSM, for Iso files succeeded.");
                    // Set the Iso domain file list fetched from VDSM into the DB.
                    refreshIsoSucceeded =
                            refreshIsoFileListMetaData(repoStorageDomainId,
                                    repoStorageDom,
                                    isoDomainList, FileTypeExtension.ISO);
                }
            } catch (Exception e) {
                refreshIsoSucceeded = false;
                log.warnFormat("The refresh process from VDSM, for Iso files failed.");
                log.error(e);
            }
        }
        return refreshIsoSucceeded;
    }

    /**
     * Gets the Iso floppy file list from VDSM, and if the fetch is valid refresh the Iso floppy list in the DB.
     *
     * @param repoFileMetaData
     *            - The repository storage pool id, we want to update the file list.
     * @param repoStorageDomainId
     *            - The repository storage domain id, for activate storage domain id.
     * @return True, if the fetch from VDSM has succeeded. False otherwise.
     */
    private boolean updateFloppyListFromVDSM(Guid repoStoragePoolId, Guid repoStorageDomainId) {
        boolean refreshFloppySucceeded = false;
        if (repoStorageDomainId != null) {
            try {
                // Get Iso domain floppy file list from storage pool.
                VDSReturnValue returnValue = Backend
                        .getInstance()
                        .getResourceManager()
                        .RunVdsCommand(VDSCommandType.GetFloppyList,
                                new IrsBaseVDSCommandParameters(repoStoragePoolId));
                @SuppressWarnings("unchecked")
                List<String> isoDomainFloppyList = (List<String>) returnValue.getReturnValue();
                if (returnValue.getSucceeded() && isoDomainFloppyList != null) {
                    // Set the Iso domain floppy file list fetched from VDSM into the DB.
                    refreshFloppySucceeded =
                            refreshIsoFileListMetaData(repoStorageDomainId,
                                    repoStorageDom,
                                    isoDomainFloppyList, FileTypeExtension.Floppy);
                }
                log.debugFormat("The refresh process from VDSM, for Floppy files succeeded.");
            } catch (Exception e) {
                refreshFloppySucceeded = false;
                log.warnFormat("The refresh process from VDSM, for Floppy files failed.");
                log.error(e);
            }
        }
        return refreshFloppySucceeded;
    }

    /**
     * Maintain a ConcurrentMap which contains Lock object.

     * The key Object is a Pair object, which will represent the domain and the file type.

     * If no synchronized object found, the Lock will be add to the ConcurrentMap.
     *
     * @param domainId
     *            - The domain Id that supposed to be refreshed.
     * @param fileTypeExt
     *            - The file type supposed to be refreshed.
     * @return - The Lock object, which represent the domain and the file type, to lock.
     */
    private static Lock getSyncObject(Guid domainId, FileTypeExtension fileTypeExt) {
        Pair<Guid, FileTypeExtension> domainPerFileType = new Pair<Guid, FileTypeExtension>(domainId, fileTypeExt);
        syncDomainForFileTypeMap.putIfAbsent(domainPerFileType,new ReentrantLock());
        return syncDomainForFileTypeMap.get(domainPerFileType);
    }

    /**
     * Add audit log message when fetch encounter problems.
     *
     * @param problematicRepoFilesList
     *            - List of Iso domain names, which encounter problem fetching from VDSM.
     */
    private void addToAuditLogErrorMessage(String problematicRepoFilesList) {
        AuditLogableBase logable = new AuditLogableBase();

        // Get translated error by error code ,if no translation found (should not happened) ,
        // will set the error code instead.
        logable.AddCustomValue("isoDomains", problematicRepoFilesList);
        AuditLogDirector.log(logable, AuditLogType.REFRESH_REPOSITORY_FILE_LIST_FAILED);
    }

    /**
     * Add audit log message when fetch encounter problems.
     */
    private void addToAuditLogSuccessMessage(String IsoDomain, String fileTypeExt) {
        AuditLogableBase logable = new AuditLogableBase();
        logable.AddCustomValue("isoDomains", String.format("%s (%s file type)", IsoDomain, fileTypeExt));
        AuditLogDirector.log(logable, AuditLogType.REFRESH_REPOSITORY_FILE_LIST_SUCCEEDED);
    }

    private static LogCompat log = LogFactoryCompat.getLog(IsoDomainListSyncronizer.class);
}


text:
const merge = require("webpack-merge");

const LodashModuleReplacementPlugin = require("lodash-webpack-plugin");

const { commonMainConfig, commonRendererConfig } = require("./webpack.common");

const mainConfig = merge(commonMainConfig, { mode: "production" });

const rendererConfig = merge(commonRendererConfig, {
  plugins: [new LodashModuleReplacementPlugin()],
  mode: "production"
});

module.exports = [mainConfig, rendererConfig];

redpajama_set_name: RedPajamaGithub


text:
module app {
    'use strict';

    angular.module('app', [
        'ngAnimate',
        'ngCookies',
        'ngTouch',
        'ngSanitize',
        'ngResource',
        'ui.materialize',

        'app.home',
        'app.components',
        'app.directives',

        'ui.router'
    ]);

    angular.module('app.home', []);
    angular.module('app.components', []);
    angular.module('app.directives', []);
}


text:
"use strict";

const fs = require(`fs-extra`);

const path = require(`path`);

class GatsbyWebpackStatsExtractor {
  constructor(options) {
    this.plugin = {
      name: `GatsbyWebpackStatsExtractor`
    };
    this.options = options || {};
  }

  apply(compiler) {
    compiler.hooks.done.tapAsync(this.plugin.name, (stats, done) => {
      let assets = {};
      let assetsMap = {};

      for (let chunkGroup of stats.compilation.chunkGroups) {
        if (chunkGroup.name) {
          let files = [];

          for (let chunk of chunkGroup.chunks) {
            files.push(...chunk.files);
          }

          assets[chunkGroup.name] = files.filter(f => f.slice(-4) !== `.map`);
          assetsMap[chunkGroup.name] = files.filter(f => f.slice(-4) !== `.map` && f.slice(0, chunkGroup.name.length) === chunkGroup.name).map(filename => `/${filename}`);
        }
      }

      const webpackStats = { ...stats.toJson({
          all: false,
          chunkGroups: true
        }),
        assetsByChunkName: assets
      };
      fs.writeFile(path.join(`public`, `chunk-map.json`), JSON.stringify(assetsMap), () => {
        fs.writeFile(path.join(`public`, `webpack.stats.json`), JSON.stringify(webpackStats), done);
      });
    });
  }

}

module.exports = GatsbyWebpackStatsExtractor;
//# sourceMappingURL=gatsby-webpack-stats-extractor.js.map


text:
.class public interface abstract Lcom/android/internal/util/IState;
.super Ljava/lang/Object;
.source "IState.java"

# static fields
.field public static final HANDLED:Z = true

.field public static final NOT_HANDLED:Z

# virtual methods
.method public abstract enter()V
.end method

.method public abstract exit()V
.end method

.method public abstract getName()Ljava/lang/String;
.end method

.method public abstract processMessage(Landroid/os/Message;)Z
.end method


text:
"Test of the Graphite JSON response object."
import json
import wordgraph
from wordgraph import grapher

import py

from tests.lib import compare

@py.test.mark.xfail
def test_server_requests_graph_structure():
    """Response data from live Graphite server filled with test data.

    Fictional data represents server requests for four fictional web servers.
    Each server's request load are approximately linear.

    http://play.grafana.org/graphite/render?from=-15min&until=now&target=aliasByNode(scaleToSeconds(apps.fakesite.*.counters.requests.count%2C1)%2C2)&format=json
    """
    with open('tests/data/server_requests.json') as data:
        graphite_data = json.load(data)

    import tests.data.server_requests
    expected_data = tests.data.server_requests.expected

    graph = grapher.GraphiteGraph()
    graphDict = {
            'title': 'server requests',
            'y_axis': {'label': 'requests per second'},
            'graphite_data': graphite_data
    }
    graph.auto_ingest(graphDict)
    structure = graph.as_dict()

    compare.assertDictionary(structure, expected_data)


text:
#include "Clipboard.h"

#include <QApplication>
#include <QClipboard>
#include <QTimer>

#include "core/Config.h"

Clipboard* Clipboard::m_instance(nullptr);

Clipboard::Clipboard(QObject* parent)
    : QObject(parent)
    , m_timer(new QTimer(this))
{
    m_timer->setSingleShot(true);
    connect(m_timer, SIGNAL(timeout()), SLOT(clearClipboard()));
    connect(qApp, SIGNAL(aboutToQuit()), SLOT(clearCopiedText()));
}

void Clipboard::setText(const QString& text)
{
    QClipboard* clipboard = QApplication::clipboard();

    clipboard->setText(text, QClipboard::Clipboard);
    if (clipboard->supportsSelection()) {
        clipboard->setText(text, QClipboard::Selection);
    }

    if (config()->get("security/clearclipboard").toBool()) {
        int timeout = config()->get("security/clearclipboardtimeout").toInt();
        if (timeout > 0) {
            m_lastCopied = text;
            m_timer->start(timeout * 1000);
        }
    }
}

void Clipboard::clearCopiedText()
{
    if (m_timer->isActive()) {
        m_timer->stop();
        clearClipboard();
    }
}

void Clipboard::clearClipboard()
{
    QClipboard* clipboard = QApplication::clipboard();

    if (!clipboard) {
        qWarning("Unable to access the clipboard.");
        return;
    }

    if (clipboard->text(QClipboard::Clipboard) == m_lastCopied) {
        clipboard->clear(QClipboard::Clipboard);
    }

    if (clipboard->supportsSelection()
            && (clipboard->text(QClipboard::Selection) == m_lastCopied)) {
        clipboard->clear(QClipboard::Selection);
    }

    m_lastCopied.clear();
}

Clipboard* Clipboard::instance()
{
    if (!m_instance) {
        m_instance = new Clipboard(qApp);
    }

    return m_instance;
}


text:
from DelvView import DelvView

class ggplot(DelvView):
  def __init__(self, svgElem):
    self.svgElem=svgElem
    self.width=20
    self.height=10

  def resize(self, width, height):
    self.width = width
    self.height = height

# TODO refactor as necessary


text:
import numpy as np
import random

class ER(object):

    def __init__(self, memory_size, state_dim, reward_dim, batch_size, history_length=1):
        self.memory_size = memory_size
        self.actions = np.random.normal(scale=0.35, size=self.memory_size)
        self.rewards = np.random.normal(scale=0.35, size=(self.memory_size, reward_dim))
        self.states = np.random.normal(scale=0.35, size=(self.memory_size, state_dim))
        self.terminals = np.zeros(self.memory_size, dtype=np.float32)
        self.batch_size = batch_size
        self.history_length = history_length
        # self.count = 0
        self.count = 200
        self.current = 0

        # pre-allocate prestates and poststates for minibatch
        self.prestates = np.empty((self.batch_size, self.history_length, state_dim), dtype = np.float32)
        self.poststates = np.empty((self.batch_size, self.history_length, state_dim), dtype = np.float32)

    def add(self, actions, rewards, states, terminals):
        # NB! state is post-state, after action and reward
        for a, r, s, t in zip(actions, rewards, states, terminals):
            self.actions[self.current] = a
            self.rewards[self.current, ...] = r
            self.states[self.current, ...] = s
            self.terminals[self.current] = t
            self.count = max(self.count, self.current + 1)
            self.current = (self.current + 1) % self.memory_size

    def get_state(self, index):
        assert self.count > 0, "replay memory is empy"
        # normalize index to expected range, allows negative indexes
        index = index % self.count
        # if is not in the beginning of matrix
        if index >= self.history_length - 1:
            # use faster slicing
            return self.states[(index - (self.history_length - 1)):(index + 1), ...]
        else:
            # otherwise normalize indexes and use slower list based access
            indexes = [(index - i) % self.count for i in reversed(range(self.history_length))]
            return self.states[indexes, ...]

    def sample(self):
        # memory must include poststate, prestate and history
        assert self.count > self.history_length
        # sample random indexes
        indexes = []
        while len(indexes) < self.batch_size:
            # find random index
            while True:
                # sample one index (ignore states wraping over
                index = random.randint(self.history_length, self.count - 1)
                # if wraps over current pointer, then get new one
                if index >= self.current and index - self.history_length < self.current:
                    continue
                # if wraps over episode end, then get new one
                # NB! poststate (last screen) can be terminal state!
                if self.terminals[(index - self.history_length):index].any():
                    continue
                # otherwise use this index
                break

            # NB! having index first is fastest in C-order matrices
            self.prestates[len(indexes), ...] = self.get_state(index - 1)
            self.poststates[len(indexes), ...] = self.get_state(index)
            indexes.append(index)

        actions = self.actions[indexes]
        rewards = self.rewards[indexes, ...]
        terminals = self.terminals[indexes]

        return self.prestates, actions, rewards, self.poststates, terminals

    def save(self):
        pass

    def load(self):
        pass


text:
using System;
using System.Web.Security;

namespace vlko.core.Authentication.Implementation
{
    public class FormsAuthenticationService : IFormsAuthenticationService
    {
        public void SignIn(string userName, bool createPersistentCookie)
        {
            if (String.IsNullOrEmpty(userName)) throw new ArgumentException("Value cannot be null or empty.", "userName");

            FormsAuthentication.SetAuthCookie(userName, createPersistentCookie);
        }

        public void SignOut()
        {
            FormsAuthentication.SignOut();
        }
    }
}


text:
package org.apache.jackrabbit.ocm.manager.inheritance;

import junit.framework.Test;
import junit.framework.TestSuite;

/**
 * Package level tests.
 *
 * @author Alexandru Popescu
 */
public class AllTests {

    public static Test suite() {
        return buildSuite();
    }

    public static Test buildSuite() {
        TestSuite suite= new TestSuite("Test for org.apache.jackrabbit.ocm.manager.inheritance");
        //$JUnit-BEGIN$
        suite.addTestSuite(AnnotationInheritanceConcreteClassTest.class);
        suite.addTestSuite(AnnotationInheritanceHierarchyTest.class);
        suite.addTestSuite(DigesterInheritanceConcreteClassTest.class);
        suite.addTestSuite(DigesterInheritanceHierarchyTest.class);
        //$JUnit-END$

        return suite;
    }
}


text:
<!DOCTYPE html>

    <meta charset="utf-8" />
    <meta http-equiv="X-UA-Compatible" content="IE=edge,chrome=1" />
    <title>Burning Man Finger</title>
    <meta name="description" content="Landing page for the Burning Man Finger" />
    <meta name="viewport" content="width=device-width" />

    <link rel="shortcut icon" href="/favicon.ico" type="image/x-icon" />
    <link rel="apple-touch-icon" href="Content/apple-touch-icon.png" />
    <link rel="apple-touch-icon" sizes="57x57" href="Content/apple-touch-icon-57x57.png" />
    <link rel="apple-touch-icon" sizes="72x72" href="Content/apple-touch-icon-72x72.png" />
    <link rel="apple-touch-icon" sizes="76x76" href="Content/apple-touch-icon-76x76.png" />
    <link rel="apple-touch-icon" sizes="114x114" href="Content/apple-touch-icon-114x114.png" />
    <link rel="apple-touch-icon" sizes="120x120" href="Content/apple-touch-icon-120x120.png" />
    <link rel="apple-touch-icon" sizes="144x144" href="Content/apple-touch-icon-144x144.png" />
    <link rel="apple-touch-icon" sizes="152x152" href="Content/apple-touch-icon-152x152.png" />
    <link rel="apple-touch-icon" sizes="180x180" href="Content/apple-touch-icon-180x180.png" />

    <link rel="stylesheet" href="Content/main.min.css" />

                Burning Man Finger


text:
/// <reference path='../typings/node/node.d.ts' />
/// <reference path='../typings/lodash/index.d.ts' />
/// <reference path='../typings/MetadataClasses.d.ts' />

import ModelBlueprint from './ModelBlueprint';
import lodash = require('lodash');
import path = require('path');
import metadata = require('MetadataClasses');
import CommonUtils from '../flexberry-common/CommonUtils';
const skipConfirmationFunc = require('../utils/skip-confirmation');

module.exports = {

  description: 'Generates an ember-data model for flexberry.',

  availableOptions: [
    { name: 'file', type: String },
    { name: 'metadata-dir', type: String },
    { name: 'skip-confirmation', type: Boolean }
  ],

  supportsAddon: function () {
    return false;
  },

  _files: null,

  isDummy: false,

  files: function () {
    if (this._files) { return this._files; }
    this.isDummy = this.options.dummy;
    let modelsDir = path.join(this.options.metadataDir, "models");
    if (!this.options.file) {
      this.options.file = this.options.entity.name + ".json";
    }
    let model: metadata.Model = ModelBlueprint.loadModel(modelsDir, this.options.file);
    if (!model.offline) {
      this._files = CommonUtils.getFilesForGeneration(this, function(v) { return v === "__root__/mixins/regenerated/serializers/__name__-offline.js"; });
    } else {
      this._files = CommonUtils.getFilesForGeneration(this);
    }
    return this._files;
  },

  afterInstall: function (options) {
    if (this.project.isEmberCLIAddon()) {
      CommonUtils.installFlexberryAddon(options, ["model", "serializer"]);
    }
  },

  processFiles(intoDir, templateVariables) {
    let skipConfirmation = this.options.skipConfirmation;
    if (skipConfirmation) {
      return skipConfirmationFunc(this, intoDir, templateVariables);
    }

    return this._super.processFiles.apply(this, [intoDir, templateVariables]);
  },

  /**
   * Blueprint Hook locals.
   * Use locals to add custom template variables. The method receives one argument: options.
   *
   * @method locals
   * @public
   *
   * @param {Object} options Options is an object containing general and entity-specific options.
   * @return {Object} Custom template variables.
   */
  locals: function(options) {
    let modelBlueprint = new ModelBlueprint(this, options);
    return lodash.defaults({
      namespace: modelBlueprint.namespace,// for use in files\__root__\mixins\regenerated\models\__name__.js
      parentModelName: modelBlueprint.parentModelName,// for use in files\__root__\mixins\regenerated\models\__name__.js
      parentClassName: modelBlueprint.parentClassName,// for use in files\__root__\mixins\regenerated\models\__name__.js
      model: modelBlueprint.model,// for use in files\__root__\mixins\regenerated\models\__name__.js
      projections: modelBlueprint.projections,// for use in files\__root__\mixins\regenerated\models\__name__.js
      validations: modelBlueprint.validations,// for use in files\__root__\mixins\regenerated\models\__name__.js
      serializerAttrs: modelBlueprint.serializerAttrs,// for use in files\__root__\mixins\regenerated\serializers\__name__.js
      offlineSerializerAttrs: modelBlueprint.offlineSerializerAttrs,// for use in files\__root__\mixins\regenerated\serializers\__name__-offline.js
      name: modelBlueprint.name,// for use in files\tests\unit\models\__name__.js, files\tests\unit\serializers\__name__.js
      needsAllModels: modelBlueprint.needsAllModels,// for use in files\tests\unit\models\__name__.js, files\tests\unit\serializers\__name__.js
      needsAllEnums: modelBlueprint.needsAllEnums,// for use in files\tests\unit\serializers\__name__.js
      needsAllObjects: modelBlueprint.needsAllObjects,// for use in files\tests\unit\serializers\__name__.js
      enumImports: modelBlueprint.enumImports,// for use in files\__root__\mixins\regenerated\models\__name__.js
      },
      modelBlueprint.lodashVariables
    );
  },

  /**
   * Blueprint Hook filesPath.
   * Override the default files directory. Useful for switching between file sets conditionally.
   *
   * @method filesPath
   * @public
   *
   * @param {Object} options Options is an object containing general and entity-specific options.
   * @return {String} Overridden files directory.
   */
  filesPath: function (options) {
    const filesSubDir = ModelBlueprint.checkCpValidations(this) ? 'files-cp-validations' : 'files-ember-validations';
    return path.join(this._super.filesPath.apply(this, [ options ]), filesSubDir);
  }
};


text:
namespace cpu {
    namespace interrupt {
        void enableArmTimer();

        void disableArmTimer();

        void enableSystemTimer(uint_fast8_t timerIdx);

        void disableSystemTimer(uint_fast8_t timerIdx);

        void enableCoreTimer();

        void disableCoreTimer();

        void enable();

        void disable();
    }
}

#endif //KERNEL_INTERRUPTS_H


text:
angular.module('gnaviApp').

  /* Prefs controller */
  controller('prefsController', function($scope, gnaviAPIservice, ngTableParams) {

    gnaviAPIservice.getPrefs().success(function (response) {
        //Digging into the response to get the relevant data
        var data = response.pref;
        $scope.tableParams = new ngTableParams({
            page: 1,            // show first page
            count:10           // count per page
        }, {
            total: data.length, // length of data
            getData: function($defer, params) {
                $defer.resolve(data.slice((params.page() - 1) * params.count(), params.page() * params.count()));
            }
        });

    });

  });


text:
package com.yxy.chukonu.reactor.java.nio.single.thread;

import java.io.IOException;
import java.net.InetSocketAddress;
import java.net.ServerSocket;
import java.nio.channels.SelectionKey;
import java.nio.channels.Selector;
import java.nio.channels.ServerSocketChannel;
import java.util.Iterator;
import java.util.Set;

public class Server implements Runnable{

	Selector selector ;
	ServerSocketChannel serverSocketChannel ;
	int port ;

	public Server(int port) {
		this.port = port ;
	}

	@Override
	public void run() {

		try {
			//1. open Selector()
			selector = Selector.open() ;
			//2. get Channel
			serverSocketChannel = ServerSocketChannel.open();
			ServerSocket serverSocket = serverSocketChannel.socket() ;
			//3. bind non-blocking mode
			serverSocketChannel.configureBlocking(false) ;
			//4. bind port
			serverSocket.bind(new InetSocketAddress(port)) ;

			System.out.println("Server is listening on port : " + port);

			//5.register ACCEPT Event
			SelectionKey event = serverSocketChannel.register(selector, SelectionKey.OP_ACCEPT) ;
			//6. bind event handler
			event.attach(new AcceptHandler()) ;

			while(!Thread.interrupted()) {
				selector.select() ;
				//7. get READY event keys
				Set<SelectionKey> selected = selector.selectedKeys() ;
				Iterator<SelectionKey> iter = selected.iterator() ;
				while(iter.hasNext()) {
					SelectionKey eventKey = iter.next() ;
					if(eventKey.isAcceptable()) {
						// this line is essential,
						// otherwise the ACCEPT event will be replayed
						serverSocketChannel.accept() ;
					}

					//8. dispatch event
					dispatch(eventKey) ;
				}

				//9. clear event list,
				//   otherwise above events will be replayed.
				selected.clear();
			}
		} catch (IOException e) {
			e.printStackTrace();
		}

	}

	private void dispatch(SelectionKey eventKey) {
		//get binded handler
		Runnable handler = (Runnable) eventKey.attachment() ;
		if(handler != null) {
			handler.run();
		}

	}

	class AcceptHandler implements Runnable{

		@Override
		public void run() {
			System.out.println("handling new connection.");

		}

	}

	public static void main(String[] args) throws Exception {
		Server server = new Server(1234) ;
		server.run();
	}

}


text:
<list model="account.move.line">
    <search>
        <field name="move_date"/>
        <field name="description"/>
        <field name="debit"/>
        <field name="credit"/>
    </search>
    <field name="move_date"/>
    <field name="description"/>
    <field name="debit"/>
    <field name="credit"/>
</list>


text:
var verasurMain = function () {

    var Initialize = function(){
        jQuery.validator.setDefaults({
            highlight: function (element) {
                $(element).closest('.form-group').addClass('has-error');
            },
            unhighlight: function (element) {
                $(element).closest('.form-group').removeClass('has-error');
            },
            errorElement: 'span',
            errorClass: 'help-block',
            errorPlacement: function (error, element) {
                if (element.parent('.input-group').length) {
                    error.insertAfter(element.parent());
                } else {
                    error.insertAfter(element);
                }
            }
        });
    };

    var redondeo = function (value, precision) {
        if (value == "" || isNaN(value)) {
            return 0;
        } else {
            return parseFloat(parseFloat(value).toFixed(precision));
        }
    };

    return {
        redondeo: redondeo,
        init: Initialize
    };

}();


text:
<?xml version="1.0" encoding="utf-8"?>
<LinearLayout
    xmlns:android="http://schemas.android.com/apk/res/android"
    xmlns:app="http://schemas.android.com/apk/res-auto"
    xmlns:tools="http://schemas.android.com/tools"
    android:layout_width="match_parent"
    android:layout_height="match_parent"
    android:orientation="vertical"
    tools:context=".activity.usercenter.SettingsActivity">

    <com.yiwen.mobike.views.TabTitleView
        android:id="@+id/title_use_detail"
        android:layout_width="match_parent"
        android:layout_height="wrap_content"
        app:titleText="设置"/>

    <include layout="@layout/divider"/>

    <com.yiwen.mobike.views.MySettingView
        android:id="@+id/tv_useaddress"
        android:layout_width="match_parent"
        android:layout_height="wrap_content"
        app:leftTvText="常用地址"
        />

    <include layout="@layout/divider"/>

    <com.yiwen.mobike.views.MySettingView
        android:id="@+id/tv_findnew"
        android:layout_width="match_parent"
        android:layout_height="wrap_content"
        app:leftTvText="检查新版本"
        />

    <include layout="@layout/divider"/>

    <com.yiwen.mobike.views.MySettingView
        android:id="@+id/tv_aboutus"
        android:layout_width="match_parent"
        android:layout_height="wrap_content"
        app:leftTvText="关于我们"
        />

    <include layout="@layout/divider_user_center"/>

    <include layout="@layout/divider"/>

    <com.yiwen.mobike.views.MySettingView
        android:id="@+id/tv_useprtool"
        android:layout_width="match_parent"
        android:layout_height="wrap_content"
        app:leftTvText="用户协议"
        />

    <include layout="@layout/divider"/>

    <com.yiwen.mobike.views.MySettingView
        android:id="@+id/tv_cashexplain"
        android:layout_width="match_parent"
        android:layout_height="wrap_content"
        app:leftTvText="押金说明"
        />

    <include layout="@layout/divider"/>

    <com.yiwen.mobike.views.MySettingView
        android:id="@+id/tv_payprtool"
        android:layout_width="match_parent"
        android:layout_height="wrap_content"
        app:leftTvText="充值协议"
        />

    <include layout="@layout/divider_user_center"/>

    <include layout="@layout/divider_user_center"/>

    <Button
        android:id="@+id/bt_logout"
        android:layout_width="match_parent"
        android:layout_height="wrap_content"
        android:layout_marginLeft="10dp"
        android:layout_marginRight="10dp"
        android:layout_marginTop="10dp"
        android:background="@color/red"
        android:text="退出登录"
        android:textColor="@color/white"/>

    <View
        android:layout_width="match_parent"
        android:layout_height="match_parent"
        android:background="@color/bg"/>
</LinearLayout>


text:
(function () {
  angular
    .module("ig.countrySelector")
    .directive("igCountrySelector", igCountrySelector);

  /* @ngInject */
  function igCountrySelector($compile, countries) {
    var directive = {
      restrict  : "A",
      transclude: true,
      link      : link
    };

    return directive;

    /////////////////////////////////////////////////////

    function link(scope, element, attributes) {
      var options,
          childElements;

      if ( !scope.selectorType )
        !attributes.selectorType ? scope.selectorType = "alpha2" : scope.selectorType = attributes.selectorType.toLowerCase();
      if ( !scope.selectorDataType )
        !attributes.selectorDataType ? scope.selectorDataType = "all" : scope.selectorDataType = attributes.selectorDataType.toLowerCase();

      switch ( scope.selectorDataType ) {
        case "itunes":
          scope.countries = countries.retrieveiTunesCountries();
          break;
        default:
          scope.countries = countries.retrieveCountries();
          break;
      }

      options       = '<option ng-repeat="country in countries" value="{{country.' + scope.selectorType + '}}">{{country.name}}</option>';
      childElements = $compile(options)(scope);

      element.append(childElements);
    }
  }
})();


text:
require 'test/unit'
require 'testutil'
require 'uri'
require 'openid/yadis/discovery'
require 'openid/fetchers'
require 'openid/util'
require 'discoverdata'

module OpenID

  module YadisDiscovery
    include FetcherMixin
    include DiscoverData

    STATUS_HEADER_RE = /Status: (\d+) .*?$/m

    def self.mkResponse(data)
      status_mo = data.scan(STATUS_HEADER_RE)
      headers_str, body = data.split("\n\n", 2)
      headers = {}
      headers_str.split("\n", -1).each { |line|
        k, v = line.split(':', 2)
        k = k.strip().downcase
        v = v.strip()
        headers[k] = v
      }
      status = status_mo[0][0].to_i
      return HTTPResponse._from_raw_data(status, body,
                                         headers)
    end

    class TestFetcher
      include DiscoverData

      def initialize(base_url)
        @base_url = base_url
      end

      def fetch(url, headers, body, redirect_limit=nil)
        current_url = url
        while true
          parsed = URI::parse(current_url)
          # parsed[2][1:]
          path = parsed.path[1..-1]
          begin
            data = generateSample(path, @base_url)
          rescue ArgumentError
            return HTTPResponse._from_raw_data(404, '', {},
                                               current_url)
          end

          response = YadisDiscovery.mkResponse(data)
          if ["301", "302", "303", "307"].member?(response.code)
            current_url = response['location']
          else
            response.final_url = current_url
            return response
          end
        end
      end
    end

    class MockFetcher
      def initialize
        @count = 0
      end

      def fetch(uri, headers=nil, body=nil, redirect_limit=nil)
        @count += 1
        if @count == 1
          headers = {
            'X-XRDS-Location'.downcase => 'http://unittest/404',
          }
          return HTTPResponse._from_raw_data(200, '', headers, uri)
        else
          return HTTPResponse._from_raw_data(404, '', {}, uri)
        end
      end
    end

    class TestSecondGet < Test::Unit::TestCase
      include FetcherMixin

      def test_404
        uri = "http://something.unittest/"
        assert_raise(DiscoveryFailure) {
          with_fetcher(MockFetcher.new) { Yadis.discover(uri) }
        }
      end
    end

    class DiscoveryTestCase
      include DiscoverData
      include FetcherMixin

      def initialize(testcase, input_name, id_name, result_name, success)
        @base_url = 'http://invalid.unittest/'
        @testcase = testcase
        @input_name = input_name
        @id_name = id_name
        @result_name = result_name
        @success = success
      end

      def setup
        @input_url, @expected = generateResult(@base_url,
                                               @input_name,
                                               @id_name,
                                               @result_name,
                                               @success)
      end

      def do_discovery
        with_fetcher(TestFetcher.new(@base_url)) do
          Yadis.discover(@input_url)
        end
      end

      def runCustomTest
        setup

        if @expected.respond_to?("ancestors") and @expected.ancestors.member?(DiscoveryFailure)
          @testcase.assert_raise(DiscoveryFailure) {
            do_discovery
          }
        else
          result = do_discovery
          @testcase.assert_equal(@input_url, result.request_uri)

          msg = sprintf("Identity URL mismatch: actual = %s, expected = %s",
                        result.normalized_uri, @expected.normalized_uri)
          @testcase.assert_equal(@expected.normalized_uri, result.normalized_uri, msg)

          msg = sprintf("Content mismatch: actual = %s, expected = %s",
                        result.response_text, @expected.response_text)
          @testcase.assert_equal(@expected.response_text, result.response_text, msg)

          expected_keys = @expected.instance_variables
          expected_keys.sort!

          actual_keys = result.instance_variables
          actual_keys.sort!

          @testcase.assert_equal(actual_keys, expected_keys)

          @expected.instance_variables.each { |k|
            exp_v = @expected.instance_variable_get(k)
            act_v = result.instance_variable_get(k)
            @testcase.assert_equal(act_v, exp_v, [k, exp_v, act_v])
          }
        end
      end
    end

    class NoContentTypeFetcher
      def fetch(url, body=nil, headers=nil, redirect_limit=nil)
        return OpenID::HTTPResponse._from_raw_data(200, "", {}, nil)
      end
    end

    class BlankContentTypeFetcher
      def fetch(url, body=nil, headers=nil, redirect_limit=nil)
        return OpenID::HTTPResponse._from_raw_data(200, "", {"Content-Type" => ""}, nil)
      end
    end

    class TestYadisDiscovery < Test::Unit::TestCase
      include FetcherMixin

      def test_yadis_discovery
        DiscoverData::TESTLIST.each { |success, input_name, id_name, result_name|
          test = DiscoveryTestCase.new(self, input_name, id_name, result_name, success)
          test.runCustomTest
        }
      end

      def test_is_xrds_yadis_location
        result = Yadis::DiscoveryResult.new('http://request.uri/')
        result.normalized_uri = "http://normalized/"
        result.xrds_uri = "http://normalized/xrds"

        assert(result.is_xrds)
      end

      def test_is_xrds_content_type
        result = Yadis::DiscoveryResult.new('http://request.uri/')
        result.normalized_uri = result.xrds_uri = "http://normalized/"
        result.content_type = Yadis::YADIS_CONTENT_TYPE

        assert(result.is_xrds)
      end

      def test_is_xrds_neither
        result = Yadis::DiscoveryResult.new('http://request.uri/')
        result.normalized_uri = result.xrds_uri = "http://normalized/"
        result.content_type = "another/content-type"

        assert(!result.is_xrds)
      end

      def test_no_content_type
        with_fetcher(NoContentTypeFetcher.new) do
          result = Yadis.discover("http://bogus")
          assert_equal(nil, result.content_type)
        end
      end

      def test_blank_content_type
        with_fetcher(BlankContentTypeFetcher.new) do
          result = Yadis.discover("http://bogus")
          assert_equal("", result.content_type)
        end
      end
    end
  end
end


text:
'use strict';

const assert = require('./../../assert');
const common = require('./../../common');

let battle;

describe('Inverse Battle', function () {
	afterEach(function () {
		battle.destroy();
	});

	it(`should change natural resistances into weaknesses`, function () {
		battle = common.createBattle({inverseMod: true}, [[
			{species: 'wynaut', moves: ['vitalthrow']},
		], [
			{species: 'scyther', moves: ['sleeptalk']},
		]]);
		battle.makeChoices();
		assert(battle.log[battle.lastMoveLine + 1].startsWith('|-supereffective|'));
	});

	it(`should change natural weaknesses into resistances`, function () {
		battle = common.createBattle({inverseMod: true}, [[
			{species: 'wynaut', moves: ['vitalthrow']},
		], [
			{species: 'absol', moves: ['sleeptalk']},
		]]);
		battle.makeChoices();
		assert(battle.log[battle.lastMoveLine + 1].startsWith('|-resisted|'));
	});

	it(`should negate natural immunities and make them weaknesses`, function () {
		battle = common.createBattle({inverseMod: true}, [[
			{species: 'wynaut', moves: ['vitalthrow']},
		], [
			{species: 'dusknoir', moves: ['sleeptalk']},
		]]);
		battle.makeChoices();
		assert(battle.log[battle.lastMoveLine + 1].startsWith('|-supereffective|'));
		assert.false.fullHP(battle.p2.active[0]);
	});

	it(`should affect Stealth Rock damage`, function () {
		battle = common.createBattle({inverseMod: true}, [[
			{species: 'wynaut', moves: ['stealthrock', 'snore']},
		], [
			{species: 'ninjask', moves: ['sleeptalk']},
			{species: 'steelix', moves: ['sleeptalk']},
			{species: 'hitmonchan', moves: ['sleeptalk']},
			{species: 'chansey', moves: ['sleeptalk']},
			{species: 'staraptor', moves: ['sleeptalk']},
			{species: 'volcarona', moves: ['sleeptalk']},
		]]);
		battle.makeChoices();
		let pokemon;
		let expectedDamage;
		for (let i = 2; i <= 6; i++) {
			battle.makeChoices('move snore', 'switch ' + i);
			pokemon = battle.p2.active[0];
			expectedDamage = Math.floor(pokemon.maxhp * Math.pow(0.5, i - 1));
			assert.equal(pokemon.maxhp - pokemon.hp, expectedDamage, `${pokemon.name} should take ${expectedDamage} damage`);
		}
	});

	it(`should affect the resistance of Delta Stream`, function () {
		battle = common.createBattle({inverseMod: true}, [[
			{species: 'wynaut', moves: ['hiddenpowerbug']},
		], [
			{species: 'rayquazamega', ability: 'deltastream', moves: ['sleeptalk']},
		]]);
		battle.makeChoices();
		assert(!battle.log[battle.lastMoveLine + 1].startsWith('|-supereffective|'));
	});

	it(`should make Ghost/Grass types take neutral damage from Flying Press`, function () {
		battle = common.createBattle({inverseMod: true}, [[
			{species: 'hawlucha', moves: ['flyingpress']},
		], [
			{species: 'gourgeist', moves: ['sleeptalk']},
		]]);
		battle.makeChoices();
		assert(!battle.log[battle.lastMoveLine + 1].startsWith('|-supereffective|'));
	});

	it(`should not affect ability-based immunities`, function () {
		battle = common.createBattle({inverseMod: true}, [[
			{species: 'wynaut', moves: ['earthquake']},
		], [
			{species: 'mismagius', ability: 'levitate', moves: ['sleeptalk']},
		]]);
		battle.makeChoices();
		assert(battle.log[battle.lastMoveLine + 1].startsWith('|-immune|'));
		assert.fullHP(battle.p2.active[0]);
	});

	it(`should not affect move-based immunities`, function () {
		battle = common.createBattle({inverseMod: true}, [[
			{species: 'wynaut', moves: ['earthquake']},
		], [
			{species: 'klefki', moves: ['magnetrise']},
		]]);
		battle.makeChoices();
		assert(battle.log[battle.lastMoveLine + 1].startsWith('|-immune|'));
		assert.fullHP(battle.p2.active[0]);
	});

	it(`should not affect the type effectiveness of Freeze Dry on Water-type Pokemon`, function () {
		battle = common.createBattle({inverseMod: true}, [[
			{species: 'wynaut', moves: ['freezedry']},
		], [
			{species: 'floatzel', moves: ['sleeptalk']},
		]]);
		battle.makeChoices();
		assert(battle.log[battle.lastMoveLine + 1].startsWith('|-supereffective|'));
	});

	it(`should not affect the "ungrounded" state of Flying-type Pokemon`, function () {
		battle = common.createBattle({inverseMod: true}, [[
			{species: 'wynaut', moves: ['spore']},
		], [
			{species: 'talonflame', moves: ['mistyterrain']},
		]]);
		battle.makeChoices();
		assert.equal(battle.p2.active[0].status, 'slp');
	});
});


text:
require 'rubygems'
require 'json'
require 'thor'
require 'thor/group'
require 'fileutils'
require 'active_support/inflector'
require 'active_support/core_ext/hash/indifferent_access'
require 'action_view'
require 'katello_api/base'
require 'net/http'
require 'uri'
require 'pp'

module KatelloApi
  module Generator

    class Base < Thor::Group
      include Thor::Actions
      include ActionView::Helpers::SanitizeHelper

      attr_reader :doc, :resource, :resource_key

      def initialize(doc, *args)
        super
        @doc = doc.first
      end

      def self.source_root
        File.expand_path("../template", __FILE__)
      end

      def self.start(katello_url, api_version = nil)
        katello_url = katello_url.sub(/\.json\Z/,"")
        katello_url << "/#{(api_version || KatelloApi::Base::API_VERSION)}.json"
        uri = URI(katello_url)
        http = Net::HTTP.new(uri.host, uri.port)
        if uri.scheme == "https"
          http.use_ssl = true
          http.verify_mode = OpenSSL::SSL::VERIFY_NONE
        end
        response = http.get(uri.path)

        unless response.code == "200"
          raise "Could not load data from #{katello_url}"
        end
        body = response.body
        File.open(KatelloApi.doc_file, "w") { |f| f.write(body) }
        doc = JSON.load(body).with_indifferent_access
        destination = File.expand_path("../../../..", __FILE__)
        super([doc[:docs]], :destination_root => destination)
      end

      def generate
        doc[:resources].each do |key, resource|
          @resource_key, @resource = key, resource
          template("resource.rb.tt", "lib/katello_api/resources/#{resource_name}.rb")
        end
      end

      protected

      def camelizer(string)
        string = string.sub(/^[a-z\d]*/) { $&.capitalize }
        string.gsub(/(?:_|(\/))([a-z\d]*)/i) { "#{$2.capitalize}" }
      end

      # Resource related helper methods:

      def resource_name
        resource[:name].gsub(/\s/, "_").downcase.singularize
      end

      def validation(method)
        stringify = lambda do |object|
          case object
            when Hash
              clone = object.dup
              object.keys.each { |key| clone[key.to_s] = stringify[clone.delete(key)] }
              clone
            when Array
              object.map { |value| stringify[value] }
            else
              object
          end
        end
        KatelloApi::Base.construct_validation_hash(stringify[method])
      end

      def format_param_description(descr, prefix = ' ')
        unless descr.empty?
          prefix + strip_tags(descr).gsub(/\n/,' ').capitalize
        end
      end

      def all_params(method)
        all_params = method[:params].dup
        method[:apis].each do |api|
          KatelloApi::Base.params_in_path(api[:api_url]).each do |path_param|
            unless all_params.any? { |param| param['name'] == path_param }
              all_params << {'name' => path_param, 'expected_type' => 'object', 'description' => %[part of +#{api[:api_url]}+ path] }
            end
          end
        end
        return all_params
      end

      def prioritize_id(param)
        return "00" if param.upcase == 'ID'
        return "00#{param}" if param =~ /_id/i
        param
      end

    end
  end
end


text:
@implementation LHConstants

NSString * const LHCloudinaryCloudName = @"lhbigday";
NSString * const LHCloudinaryAPIKey = @"619553892666764";
NSString * const LHCloudinaryAPISecret = @"uvzeUVZDimFMIwPMjAO3xPDhIUE";
NSString * const LHFireBaseURL = @"https://luminous-fire-3877.firebaseio.com";

@end


text:
// Copyright 2016, Google, Inc.
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//    http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

'use strict';

// [START app]
// [START import_libraries]
var gcloud = require('gcloud');
// [END import_libraries]

// [START authenticate]
// By default, gcloud will authenticate using the service account file specified
// by the GOOGLE_APPLICATION_CREDENTIALS environment variable and use the
// project specified by the GCLOUD_PROJECT environment variable. See
// https://googlecloudplatform.github.io/gcloud-node/#/docs/guides/authentication

// Get a reference to the vision component
var vision = gcloud.vision();
// [END authenticate]

/**
 * Uses the Vision API to detect labels in the given file.
 */
// [START construct_request]
function detectLabels (inputFile, callback) {
  // Make a call to the Vision API to detect the labels
  vision.detectLabels(inputFile, { verbose: true }, function (err, labels) {
    if (err) {
      return callback(err);
    }
    console.log('result:', JSON.stringify(labels, null, 2));
    callback(null, labels);
  });
}
// [END construct_request]

// Run the example
function main (inputFile, callback) {
  detectLabels(inputFile, function (err, labels) {
    if (err) {
      return callback(err);
    }

    // [START parse_response]
    console.log('Found label: ' + labels[0].desc + ' for ' + inputFile);
    // [END parse_response]
    callback(null, labels);
  });
}

// [START run_application]
if (module === require.main) {
  if (process.argv.length < 3) {
    console.log('Usage: node labelDetection <inputFile>');
    process.exit(1);
  }
  var inputFile = process.argv[2];
  main(inputFile, console.log);
}
// [END run_application]
// [END app]

exports.main = main;


text:
package com.davidbracewell.hermes.lexicon;

import com.davidbracewell.Tag;
import com.davidbracewell.conversion.Cast;
import com.davidbracewell.function.SerializablePredicate;
import com.davidbracewell.guava.common.base.Preconditions;
import com.davidbracewell.guava.common.primitives.Doubles;
import com.davidbracewell.hermes.AttributeType;
import com.davidbracewell.hermes.HString;
import com.davidbracewell.hermes.extraction.regex.QueryToPredicate;
import com.davidbracewell.io.CSV;
import com.davidbracewell.io.CSVReader;
import com.davidbracewell.io.resource.Resource;
import com.davidbracewell.logging.Logger;
import com.davidbracewell.parsing.ParseException;
import lombok.Builder;
import lombok.Data;

import java.io.Serializable;

/**
 * The type Lexicon spec.
 *
 * @author David B. Bracewell
 */
@Data
public class LexiconSpec implements Serializable {
   private static final Logger log = Logger.getLogger(LexiconSpec.class);

   private static final long serialVersionUID = 1L;
   private boolean probabilistic;
   private boolean hasConstraints;
   private boolean caseSensitive;
   private AttributeType tagAttribute;
   private Object tag;
   private boolean useResourceNameAsTag;
   private Resource resource;

   /**
    * Instantiates a new Lexicon spec.
    */
   public LexiconSpec() {
      this(false, false, false, null, null, false, null);
   }

   /**
    * Instantiates a new Lexicon spec.
    *
    * @param caseSensitive        the case sensitive
    * @param hasConstraints       the has constraints
    * @param probabilistic        the probabilistic
    * @param resource             the resource
    * @param tagAttribute         the tag attribute
    * @param useResourceNameAsTag the use resource name as tag
    */
   @Builder
   public LexiconSpec(boolean caseSensitive, boolean hasConstraints, boolean probabilistic, Resource resource, AttributeType tagAttribute, boolean useResourceNameAsTag, Object tag) {
      this.caseSensitive = caseSensitive;
      this.hasConstraints = hasConstraints;
      this.probabilistic = probabilistic;
      this.resource = resource;
      this.tagAttribute = tagAttribute;
      this.useResourceNameAsTag = useResourceNameAsTag;
      this.tag = tag;
   }

   /**
    * Create lexicon.
    *
    * @return the lexicon
    * @throws Exception the exception
    */
   public Lexicon create() throws Exception {
      Lexicon lexicon = new TrieLexicon(caseSensitive, probabilistic, tagAttribute);
      if (resource != null) {
         String base = resource.baseName().replaceFirst("\\.[^\\.]*$", "");
         try (CSVReader reader = CSV.builder().removeEmptyCells().reader(resource)) {
            reader.forEach(row -> {
               if (row.size() > 0) {
                  String lemma = row.get(0);
                  double probability = 1d;
                  Tag tag = null;
                  SerializablePredicate<HString> constraint = null;

                  int nc = 1;

                  if (tagAttribute != null && this.tag != null) {
                     tag = Cast.as(this.tag);
                  } else if (row.size() > nc && tagAttribute != null && !useResourceNameAsTag) {
                     tag = tagAttribute.getValueType().decode(row.get(nc));
                     if (tag == null) {
                        log.warn("{0} is an invalid {1}, skipping entry {2}.", row.get(nc), tagAttribute.name(), row);
                        return;
                     }
                     nc++;
                  } else if (tagAttribute != null) {
                     tag = tagAttribute.getValueType().decode(base);
                     Preconditions.checkNotNull(tag, base + " is an invalid tag.");
                  }

                  if (probabilistic && row.size() > nc && Doubles.tryParse(row.get(nc)) != null) {
                     probability = Double.parseDouble(row.get(nc));
                     nc++;
                  }

                  if (hasConstraints && row.size() > nc) {
                     try {
                        constraint = QueryToPredicate.parse(row.get(nc));
                     } catch (ParseException e) {
                        if (tag == null) {
                           log.warn("Error parsing constraint {0}, skipping entry {1}.", row.get(nc), row);
                           return;
                        }
                     }
                  }

                  lexicon.add(new LexiconEntry(lemma, probability, constraint, tag));
               }
            });
         }
      }
      return lexicon;
   }

}//END OF LexiconSpec


text:
module Gitlab
  module Artifacts
    class MigrationHelper
      def migrate_to_remote_storage(&block)
        artifacts = ::Ci::JobArtifact.with_files_stored_locally
        migrate(artifacts, ObjectStorage::Store::REMOTE, &block)
      end

      def migrate_to_local_storage(&block)
        artifacts = ::Ci::JobArtifact.with_files_stored_remotely
        migrate(artifacts, ObjectStorage::Store::LOCAL, &block)
      end

      private

      def batch_size
        ENV.fetch('MIGRATION_BATCH_SIZE', 10).to_i
      end

      def migrate(artifacts, store, &block)
        artifacts.find_each(batch_size: batch_size) do |artifact| # rubocop:disable CodeReuse/ActiveRecord
          artifact.file.migrate!(store)

          yield artifact if block
        rescue => e
          raise StandardError.new("Failed to transfer artifact of type #{artifact.file_type} and ID #{artifact.id} with error: #{e.message}")
        end
      end
    end
  end
end


text:
#pragma once

#include <cstdint>
#include <functional>
#include <memory>
#include <string>

#include "envoy/api/os_sys_calls.h"
#include "envoy/common/exception.h"
#include "envoy/common/platform.h"
#include "envoy/common/pure.h"
#include "envoy/network/io_handle.h"

#include "common/common/byte_order.h"

#include "absl/container/inlined_vector.h"
#include "absl/strings/string_view.h"
#include "absl/types/optional.h"

namespace Envoy {
namespace Buffer {

/**
 * A raw memory data slice including location and length.
 */
struct RawSlice {
  void* mem_ = nullptr;
  size_t len_ = 0;

  bool operator==(const RawSlice& rhs) const { return mem_ == rhs.mem_ && len_ == rhs.len_; }
};

using RawSliceVector = absl::InlinedVector<RawSlice, 16>;

/**
 * A wrapper class to facilitate passing in externally owned data to a buffer via addBufferFragment.
 * When the buffer no longer needs the data passed in through a fragment, it calls done() on it.
 */
class BufferFragment {
public:
  virtual ~BufferFragment() = default;
  /**
   * @return const void* a pointer to the referenced data.
   */
  virtual const void* data() const PURE;

  /**
   * @return size_t the size of the referenced data.
   */
  virtual size_t size() const PURE;

  /**
   * Called by a buffer when the referenced data is no longer needed.
   */
  virtual void done() PURE;
};

/**
 * A basic buffer abstraction.
 */
class Instance {
public:
  virtual ~Instance() = default;

  /**
   * Copy data into the buffer (deprecated, use absl::string_view variant
   * instead).
   * TODO(htuch): Cleanup deprecated call sites.
   * @param data supplies the data address.
   * @param size supplies the data size.
   */
  virtual void add(const void* data, uint64_t size) PURE;

  /**
   * Add externally owned data into the buffer. No copying is done. fragment is not owned. When
   * the fragment->data() is no longer needed, fragment->done() is called.
   * @param fragment the externally owned data to add to the buffer.
   */
  virtual void addBufferFragment(BufferFragment& fragment) PURE;

  /**
   * Copy a string into the buffer.
   * @param data supplies the string to copy.
   */
  virtual void add(absl::string_view data) PURE;

  /**
   * Copy another buffer into this buffer.
   * @param data supplies the buffer to copy.
   */
  virtual void add(const Instance& data) PURE;

  /**
   * Prepend a string_view to the buffer.
   * @param data supplies the string_view to copy.
   */
  virtual void prepend(absl::string_view data) PURE;

  /**
   * Prepend data from another buffer to this buffer.
   * The supplied buffer is drained after this operation.
   * @param data supplies the buffer to copy.
   */
  virtual void prepend(Instance& data) PURE;

  /**
   * Commit a set of slices originally obtained from reserve(). The number of slices should match
   * the number obtained from reserve(). The size of each slice can also be altered. Commit must
   * occur once following a reserve() without any mutating operations in between other than to the
   * iovecs len_ fields.
   * @param iovecs supplies the array of slices to commit.
   * @param num_iovecs supplies the size of the slices array.
   */
  virtual void commit(RawSlice* iovecs, uint64_t num_iovecs) PURE;

  /**
   * Copy out a section of the buffer.
   * @param start supplies the buffer index to start copying from.
   * @param size supplies the size of the output buffer.
   * @param data supplies the output buffer to fill.
   */
  virtual void copyOut(size_t start, uint64_t size, void* data) const PURE;

  /**
   * Drain data from the buffer.
   * @param size supplies the length of data to drain.
   */
  virtual void drain(uint64_t size) PURE;

  /**
   * Fetch the raw buffer slices.
   * @param max_slices supplies an optional limit on the number of slices to fetch, for performance.
   * @return RawSliceVector with non-empty slices in the buffer.
   */
  virtual RawSliceVector
  getRawSlices(absl::optional<uint64_t> max_slices = absl::nullopt) const PURE;

  /**
   * @return uint64_t the total length of the buffer (not necessarily contiguous in memory).
   */
  virtual uint64_t length() const PURE;

  /**
   * @return a pointer to the first byte of data that has been linearized out to size bytes.
   */
  virtual void* linearize(uint32_t size) PURE;

  /**
   * Move a buffer into this buffer. As little copying is done as possible.
   * @param rhs supplies the buffer to move.
   */
  virtual void move(Instance& rhs) PURE;

  /**
   * Move a portion of a buffer into this buffer. As little copying is done as possible.
   * @param rhs supplies the buffer to move.
   * @param length supplies the amount of data to move.
   */
  virtual void move(Instance& rhs, uint64_t length) PURE;

  /**
   * Read from a file descriptor directly into the buffer.
   * @param io_handle supplies the io handle to read from.
   * @param max_length supplies the maximum length to read.
   * @return a IoCallUint64Result with err_ = nullptr and rc_ = the number of bytes
   * read if successful, or err_ = some IoError for failure. If call failed, rc_ shouldn't be used.
   */
  virtual Api::IoCallUint64Result read(Network::IoHandle& io_handle, uint64_t max_length) PURE;

  /**
   * Reserve space in the buffer.
   * @param length supplies the amount of space to reserve.
   * @param iovecs supplies the slices to fill with reserved memory.
   * @param num_iovecs supplies the size of the slices array.
   * @return the number of iovecs used to reserve the space.
   */
  virtual uint64_t reserve(uint64_t length, RawSlice* iovecs, uint64_t num_iovecs) PURE;

  /**
   * Search for an occurrence of data within the buffer.
   * @param data supplies the data to search for.
   * @param size supplies the length of the data to search for.
   * @param start supplies the starting index to search from.
   * @return the index where the match starts or -1 if there is no match.
   */
  virtual ssize_t search(const void* data, uint64_t size, size_t start) const PURE;

  /**
   * Search for an occurrence of data at the start of a buffer.
   * @param data supplies the data to search for.
   * @return true if this buffer starts with data, false otherwise.
   */
  virtual bool startsWith(absl::string_view data) const PURE;

  /**
   * Constructs a flattened string from a buffer.
   * @return the flattened string.
   */
  virtual std::string toString() const PURE;

  /**
   * Write the buffer out to a file descriptor.
   * @param io_handle supplies the io_handle to write to.
   * @return a IoCallUint64Result with err_ = nullptr and rc_ = the number of bytes
   * written if successful, or err_ = some IoError for failure. If call failed, rc_ shouldn't be
   * used.
   */
  virtual Api::IoCallUint64Result write(Network::IoHandle& io_handle) PURE;

  /**
   * Copy an integer out of the buffer.
   * @param start supplies the buffer index to start copying from.
   * @param Size how many bytes to read out of the buffer.
   * @param Endianness specifies the byte order to use when decoding the integer.
   * @details Size parameter: Some protocols have integer fields whose size in bytes won't match the
   * size in bytes of C++'s integer types. Take a 3-byte integer field for example, which we want to
   * represent as a 32-bit (4 bytes) integer. One option to deal with that situation is to read 4
   * bytes from the buffer and ignore 1. There are a few problems with that solution, though.
   *   * The first problem is buffer underflow: there may not be more than Size bytes available
   * (say, last field in the payload), so that's an edge case to take into consideration.
   *   * The second problem is draining the buffer after reading. With the above solution we cannot
   *     read and discard in one go. We'd need to peek 4 bytes, ignore 1 and then drain 3. That not
   *     only looks hacky since the sizes don't match, but also produces less terse code and
   * requires the caller to propagate that logic to all call sites. Things complicate even further
   * when endianness is taken into consideration: should the most or least-significant bytes be
   * padded? Dealing with this situation requires a high level of care and attention to detail.
   * Properly calculating which bytes to discard and how to displace the data is not only error
   * prone, but also shifts to the caller a burden that could be solved in a much more generic,
   * transparent and well tested manner.
   *   * The last problem in the list is sign extension, which should be properly handled when
   * reading signed types with negative values. To make matters easier, the optional Size parameter
   * can be specified in those situations where there's a need to read less bytes than a C++'s
   * integer size in bytes. For the most common case when one needs to read exactly as many bytes as
   * the size of C++'s integer, this parameter can simply be omitted and it will be automatically
   * deduced from the size of the type T
   */
  template <typename T, ByteOrder Endianness = ByteOrder::Host, size_t Size = sizeof(T)>
  T peekInt(uint64_t start = 0) {
    static_assert(Size <= sizeof(T), "requested size is bigger than integer being read");

    if (length() < start + Size) {
      throw EnvoyException("buffer underflow");
    }

    constexpr const auto displacement = Endianness == ByteOrder::BigEndian ? sizeof(T) - Size : 0;

    auto result = static_cast<T>(0);
    constexpr const auto all_bits_enabled = static_cast<T>(~static_cast<T>(0));

    int8_t* bytes = reinterpret_cast<int8_t*>(std::addressof(result));
    copyOut(start, Size, &bytes[displacement]);

    constexpr const auto most_significant_read_byte =
        Endianness == ByteOrder::BigEndian ? displacement : Size - 1;

    // If Size == sizeof(T), we need to make sure we don't generate an invalid left shift
    // (e.g. int32 << 32), even though we know that that branch of the conditional will.
    // not be taken. Size % sizeof(T) gives us the correct left shift when Size < sizeof(T),
    // and generates a left shift of 0 bits when Size == sizeof(T)
    const auto sign_extension_bits =
        std::is_signed<T>::value && Size < sizeof(T) && bytes[most_significant_read_byte] < 0
            ? static_cast<T>(static_cast<typename std::make_unsigned<T>::type>(all_bits_enabled)
                             << ((Size % sizeof(T)) * CHAR_BIT))
            : static_cast<T>(0);

    return fromEndianness<Endianness>(static_cast<T>(result)) | sign_extension_bits;
  }

  /**
   * Copy a little endian integer out of the buffer.
   * @param start supplies the buffer index to start copying from.
   * @param Size how many bytes to read out of the buffer.
   */
  template <typename T, size_t Size = sizeof(T)> T peekLEInt(uint64_t start = 0) {
    return peekInt<T, ByteOrder::LittleEndian, Size>(start);
  }

  /**
   * Copy a big endian integer out of the buffer.
   * @param start supplies the buffer index to start copying from.
   * @param Size how many bytes to read out of the buffer.
   */
  template <typename T, size_t Size = sizeof(T)> T peekBEInt(uint64_t start = 0) {
    return peekInt<T, ByteOrder::BigEndian, Size>(start);
  }

  /**
   * Copy an integer out of the buffer and drain the read data.
   * @param Size how many bytes to read out of the buffer.
   * @param Endianness specifies the byte order to use when decoding the integer.
   */
  template <typename T, ByteOrder Endianness = ByteOrder::Host, size_t Size = sizeof(T)>
  T drainInt() {
    const auto result = peekInt<T, Endianness, Size>();
    drain(Size);
    return result;
  }

  /**
   * Copy a little endian integer out of the buffer and drain the read data.
   * @param Size how many bytes to read out of the buffer.
   */
  template <typename T, size_t Size = sizeof(T)> T drainLEInt() {
    return drainInt<T, ByteOrder::LittleEndian, Size>();
  }

  /**
   * Copy a big endian integer out of the buffer and drain the read data.
   * @param Size how many bytes to read out of the buffer.
   */
  template <typename T, size_t Size = sizeof(T)> T drainBEInt() {
    return drainInt<T, ByteOrder::BigEndian, Size>();
  }

  /**
   * Copy a byte into the buffer.
   * @param value supplies the byte to copy into the buffer.
   */
  void writeByte(uint8_t value) { add(std::addressof(value), 1); }

  /**
   * Copy value as a byte into the buffer.
   * @param value supplies the byte to copy into the buffer.
   */
  template <typename T> void writeByte(T value) { writeByte(static_cast<uint8_t>(value)); }

  /**
   * Copy an integer into the buffer.
   * @param value supplies the integer to copy into the buffer.
   * @param Size how many bytes to write from the requested integer.
   * @param Endianness specifies the byte order to use when encoding the integer.
   */
  template <ByteOrder Endianness = ByteOrder::Host, typename T, size_t Size = sizeof(T)>
  void writeInt(T value) {
    static_assert(Size <= sizeof(T), "requested size is bigger than integer being written");

    const auto data = toEndianness<Endianness>(value);
    constexpr const auto displacement = Endianness == ByteOrder::BigEndian ? sizeof(T) - Size : 0;
    add(reinterpret_cast<const char*>(std::addressof(data)) + displacement, Size);
  }

  /**
   * Copy an integer into the buffer in little endian byte order.
   * @param value supplies the integer to copy into the buffer.
   * @param Size how many bytes to write from the requested integer.
   */
  template <typename T, size_t Size = sizeof(T)> void writeLEInt(T value) {
    writeInt<ByteOrder::LittleEndian, T, Size>(value);
  }

  /**
   * Copy an integer into the buffer in big endian byte order.
   * @param value supplies the integer to copy into the buffer.
   * @param Size how many bytes to write from the requested integer.
   */
  template <typename T, size_t Size = sizeof(T)> void writeBEInt(T value) {
    writeInt<ByteOrder::BigEndian, T, Size>(value);
  }
};

using InstancePtr = std::unique_ptr<Instance>;

/**
 * A factory for creating buffers which call callbacks when reaching high and low watermarks.
 */
class WatermarkFactory {
public:
  virtual ~WatermarkFactory() = default;

  /**
   * Creates and returns a unique pointer to a new buffer.
   * @param below_low_watermark supplies a function to call if the buffer goes under a configured
   *   low watermark.
   * @param above_high_watermark supplies a function to call if the buffer goes over a configured
   *   high watermark.
   * @return a newly created InstancePtr.
   */
  virtual InstancePtr create(std::function<void()> below_low_watermark,
                             std::function<void()> above_high_watermark) PURE;
};

using WatermarkFactoryPtr = std::unique_ptr<WatermarkFactory>;

} // namespace Buffer
} // namespace Envoy