gt: 				if (id != null)
context: package org.molgenis.data.importer;
  
  import static com.google.common.base.Predicates.notNull;
  import static com.google.common.collect.FluentIterable.from;
  import static java.util.Objects.requireNonNull;
  import static java.util.stream.Collectors.toList;
  import static java.util.stream.StreamSupport.stream;
  import static org.molgenis.security.core.runas.RunAsSystemProxy.runAsSystem;
  
  import java.sql.Timestamp;
  import java.text.ParseException;
  import java.util.ArrayList;
  import java.util.Arrays;
  import java.util.Collection;
  import java.util.Collections;
  import java.util.Iterator;
  import java.util.List;
  import java.util.Map;
  import java.util.Set;
  import java.util.stream.Collectors;
  
  import org.apache.commons.io.IOUtils;
  import org.molgenis.MolgenisFieldTypes.FieldTypeEnum;
  import org.molgenis.data.AttributeMetaData;
  import org.molgenis.data.DataConverter;
  import org.molgenis.data.DataService;
  import org.molgenis.data.DatabaseAction;
  import org.molgenis.data.Entity;
  import org.molgenis.data.EntityMetaData;
  import org.molgenis.data.MolgenisDataAccessException;
  import org.molgenis.data.MolgenisDataException;
  import org.molgenis.data.Package;
  import org.molgenis.data.Query;
  import org.molgenis.data.Repository;
  import org.molgenis.data.RepositoryCapability;
  import org.molgenis.data.RepositoryCollection;
  import org.molgenis.data.UnknownAttributeException;
  import org.molgenis.data.UnknownEntityException;
  import org.molgenis.data.i18n.I18nStringMetaData;
  import org.molgenis.data.i18n.LanguageMetaData;
  import org.molgenis.data.meta.TagMetaData;
  import org.molgenis.data.semantic.LabeledResource;
  import org.molgenis.data.semantic.Tag;
  import org.molgenis.data.semanticsearch.service.TagService;
  import org.molgenis.data.support.ConvertingIterable;
  import org.molgenis.data.support.DefaultEntity;
  import org.molgenis.data.support.EntityMetaDataUtils;
  import org.molgenis.data.support.LazyEntity;
  import org.molgenis.data.support.QueryImpl;
  import org.molgenis.data.validation.MolgenisValidationException;
  import org.molgenis.fieldtypes.FieldType;
  import org.molgenis.fieldtypes.IntField;
  import org.molgenis.fieldtypes.MrefField;
  import org.molgenis.fieldtypes.StringField;
  import org.molgenis.fieldtypes.XrefField;
  import org.molgenis.framework.db.EntityImportReport;
  import org.molgenis.security.core.MolgenisPermissionService;
  import org.molgenis.security.core.Permission;
  import org.molgenis.security.core.runas.RunAsSystem;
  import org.molgenis.security.core.utils.SecurityUtils;
  import org.molgenis.security.permission.PermissionSystemService;
  import org.molgenis.util.DependencyResolver;
  import org.molgenis.util.HugeSet;
  import org.molgenis.util.MolgenisDateFormat;
  import org.slf4j.Logger;
  import org.slf4j.LoggerFactory;
  import org.springframework.security.core.context.SecurityContextHolder;
  import org.springframework.transaction.annotation.Isolation;
  import org.springframework.transaction.annotation.Transactional;
  
  import com.google.common.base.Function;
  import com.google.common.base.Predicate;
  import com.google.common.collect.ImmutableMap;
  import com.google.common.collect.Iterables;
  import com.google.common.collect.Iterators;
  import com.google.common.collect.Lists;
  import com.google.common.collect.Sets;
  
  /**
   * Writes the imported metadata and data to target {@link RepositoryCollection}.
   */
  public class ImportWriter
  {
  	private static final Logger LOG = LoggerFactory.getLogger(ImportWriter.class);
  
  	private final DataService dataService;
  	private final PermissionSystemService permissionSystemService;
  	private final TagService<LabeledResource, LabeledResource> tagService;
  	private final MolgenisPermissionService molgenisPermissionService;
  
  	/**
  	 * Creates the ImportWriter
  	 * 
  	 * @param dataService
  	 *            {@link DataService} to query existing repositories and transform entities
  	 * @param permissionSystemService
  	 *            {@link PermissionSystemService} to give permissions on uploaded entities
  	 */
  	public ImportWriter(DataService dataService, PermissionSystemService permissionSystemService,
  			TagService<LabeledResource, LabeledResource> tagService,
  			MolgenisPermissionService molgenisPermissionService)
  	{
  		this.dataService = dataService;
  		this.permissionSystemService = permissionSystemService;
  		this.tagService = tagService;
  		this.molgenisPermissionService = molgenisPermissionService;
  	}
  
  	// Use transaction isolation level SERIALIZABLE to prevent problems with the async template, to do ddl statements
  	// without influencing on the current transaction
  	@Transactional(isolation = Isolation.SERIALIZABLE)
  	public EntityImportReport doImport(EmxImportJob job)
  	{
  		// languages first
  		importLanguages(job.report, job.parsedMetaData.getLanguages(), job.dbAction, job.metaDataChanges);
  
  		runAsSystem(() -> importTags(job.source));
  		importPackages(job.parsedMetaData);
  		addEntityMetaData(job.parsedMetaData, job.report, job.metaDataChanges);
  		addEntityPermissions(job.metaDataChanges);
  		runAsSystem(() -> importEntityAndAttributeTags(job.parsedMetaData));
  		importData(job.report, job.parsedMetaData.getEntities(), job.source, job.dbAction, job.defaultPackage);
  		importI18nStrings(job.report, job.parsedMetaData.getI18nStrings(), job.dbAction);
  
  		return job.report;
  	}
  
  	private void importLanguages(EntityImportReport report, Map<String, Entity> languages, DatabaseAction dbAction,
  			MetaDataChanges metaDataChanges)
  	{
  		if (!languages.isEmpty())
  		{
  			Repository repo = dataService.getRepository(LanguageMetaData.ENTITY_NAME);
  
  			List<Entity> transformed = languages.values().stream()
  					.map(e -> new DefaultEntityImporter(repo.getEntityMetaData(), dataService, e, false))
  					.collect(toList());
  
  			// Find new ones
  			transformed.stream().map(Entity::getIdValue).forEach(id -> {
  				if (repo.findOne(id) == null)
  				{
  					metaDataChanges.addLanguage(languages.get(id));
  				}
  			});
  
  			int count = update(repo, transformed, dbAction);
  			report.addEntityCount(LanguageMetaData.ENTITY_NAME, count);
  		}
  	}
  
  	private void importI18nStrings(EntityImportReport report, Map<String, Entity> i18nStrings, DatabaseAction dbAction)
  	{
  		if (!i18nStrings.isEmpty())
  		{
  			Repository repo = dataService.getRepository(I18nStringMetaData.ENTITY_NAME);
  
  			List<Entity> transformed = i18nStrings.values().stream()
  					.map(e -> new DefaultEntityImporter(I18nStringMetaData.INSTANCE, dataService, e, false))
  					.collect(toList());
  
  			int count = update(repo, transformed, dbAction);
  			report.addEntityCount(I18nStringMetaData.ENTITY_NAME, count);
  		}
  	}
  
  	private void importEntityAndAttributeTags(ParsedMetaData parsedMetaData)
  	{
  		for (Tag<EntityMetaData, LabeledResource, LabeledResource> tag : parsedMetaData.getEntityTags())
  		{
  			tagService.addEntityTag(tag);
  		}
  
  		for (EntityMetaData emd : parsedMetaData.getAttributeTags().keySet())
  		{
  			for (Tag<AttributeMetaData, LabeledResource, LabeledResource> tag : parsedMetaData.getAttributeTags()
  					.get(emd))
  			{
  				tagService.addAttributeTag(emd, tag);
  			}
  		}
  	}
  
  	/**
  	 * Imports entity data for all entities in {@link #resolved} from {@link #source}
  	 */
  	private void importData(EntityImportReport report, Iterable<EntityMetaData> resolved, RepositoryCollection source,
  			DatabaseAction dbAction, String defaultPackage)
  	{
  		for (final EntityMetaData entityMetaData : resolved)
  		{
  			String name = entityMetaData.getName();
  
  			// Languages and i18nstrings are already done
  			if (!name.equalsIgnoreCase(LanguageMetaData.ENTITY_NAME)
  					&& !name.equalsIgnoreCase(I18nStringMetaData.ENTITY_NAME) && dataService.hasRepository(name))
  			{
  				Repository repository = dataService.getRepository(name);
  				Repository fileEntityRepository = source.getRepository(entityMetaData.getName());
  
  				// Try without default package
  				if ((fileEntityRepository == null) && (defaultPackage != null)
  						&& entityMetaData.getName().toLowerCase().startsWith(defaultPackage.toLowerCase() + "_"))
  				{
  					fileEntityRepository = source
  							.getRepository(entityMetaData.getName().substring(defaultPackage.length() + 1));
  				}
  
  				// check to prevent nullpointer when importing metadata only
  				if (fileEntityRepository != null)
  				{
  					boolean selfReferencing = DependencyResolver.hasSelfReferences(entityMetaData);
  
  					// transforms entities so that they match the entity meta data of the output repository
  					Iterable<Entity> entities = Iterables.transform(fileEntityRepository, new Function<Entity, Entity>()
  					{
  						@Override
  						public Entity apply(Entity entity)
  						{
  							return new DefaultEntityImporter(entityMetaData, dataService, entity, selfReferencing);
  						}
  					});
  
  					if (selfReferencing)
  					{
  						entities = new DependencyResolver().resolveSelfReferences(entities, entityMetaData);
  					}
  					int count = update(repository, entities, dbAction);
  					if (selfReferencing && dbAction != DatabaseAction.UPDATE)
  					{
  						// Fix self referenced entities were not imported
  						update(repository, this.keepSelfReferencedEntities(entities), DatabaseAction.UPDATE);
  					}
  					report.addEntityCount(name, count);
  				}
  			}
  		}
  	}
  
  	/**
  	 * Keeps the entities that have: 1. A reference to themselves. 2. Minimal one value.
  	 * 
  	 * @param entities
  	 * @return Iterable<Entity> - filtered entities;
  	 */
  	private Iterable<Entity> keepSelfReferencedEntities(Iterable<Entity> entities)
  	{
  		return Iterables.filter(entities, new Predicate<Entity>()
  		{
  			@Override
  			public boolean apply(Entity entity)
  			{
  				Iterator<AttributeMetaData> attributes = entity.getEntityMetaData().getAttributes().iterator();
  				while (attributes.hasNext())
  				{
  					AttributeMetaData attribute = attributes.next();
  					if (attribute.getRefEntity() != null
  							&& attribute.getRefEntity().getName().equals(entity.getEntityMetaData().getName()))
  					{
  						List<String> ids = entity.getList(attribute.getName());
  						Iterable<Entity> refEntities = entity.getEntities(attribute.getName());
  						if (ids != null && ids.size() != Iterators.size(refEntities.iterator()))
  						{
  							throw new UnknownEntityException(
  									"One or more values [" + ids + "] from " + attribute.getDataType() + " field "
  											+ attribute.getName() + " could not be resolved");
  						}
  						return true;
  					}
  				}
  
  				return false;
  			}
  		});
  	}
  
  	/**
  	 * Gives the user permission to see and edit his imported entities, unless the user is admin since admins can do
  	 * that anyways.
  	 */
  	private void addEntityPermissions(MetaDataChanges metaDataChanges)
  	{
  		if (!SecurityUtils.currentUserIsSu())
  		{
  			permissionSystemService.giveUserEntityPermissions(SecurityContextHolder.getContext(),
  					metaDataChanges.getAddedEntities());
  		}
  	}
  
  	/**
  	 * Adds the parsed {@link ParsedMetaData}, creating new repositories where necessary.
  	 */
  	private void addEntityMetaData(ParsedMetaData parsedMetaData, EntityImportReport report,
  			MetaDataChanges metaDataChanges)
  	{
  		for (EntityMetaData entityMetaData : parsedMetaData.getEntities())
  		{
  			String name = entityMetaData.getName();
  			if (!EmxMetaDataParser.ENTITIES.equals(name) && !EmxMetaDataParser.ATTRIBUTES.equals(name)
  					&& !EmxMetaDataParser.PACKAGES.equals(name) && !EmxMetaDataParser.TAGS.equals(name)
  					&& !EmxMetaDataParser.LANGUAGES.equals(name) && !EmxMetaDataParser.I18NSTRINGS.equals(name))
  			{
  				if (dataService.getMeta().getEntityMetaData(entityMetaData.getName()) == null)
  				{
  					LOG.debug("trying to create: " + name);
  					metaDataChanges.addEntity(name);
  					Repository repo = dataService.getMeta().addEntityMeta(entityMetaData);
  					if (repo != null)
  					{
  						report.addNewEntity(name);
  					}
  				}
  				else if (!entityMetaData.isAbstract())
  				{
  					List<AttributeMetaData> addedAttributes = dataService.getMeta().updateEntityMeta(entityMetaData);
  					metaDataChanges.addAttributes(name, addedAttributes);
  				}
  			}
  		}
  	}
  
  	/**
  	 * Adds the packages from the packages sheet to the {@link #metaDataService}.
  	 */
  	private void importPackages(ParsedMetaData parsedMetaData)
  	{
  		for (Package p : parsedMetaData.getPackages().values())
  		{
  			if (p != null)
  			{
  				dataService.getMeta().addPackage(p);
  			}
  		}
  	}
  
  	/**
  	 * Imports the tags from the tag sheet.
  	 */
  	// FIXME: can everybody always update a tag?
  	private void importTags(RepositoryCollection source)
  	{
  		Repository tagRepo = source.getRepository(TagMetaData.ENTITY_NAME);
  		if (tagRepo != null)
  		{
  			for (Entity tag : tagRepo)
  			{
  				Entity transformed = new DefaultEntity(TagMetaData.INSTANCE, dataService, tag);
  				Entity existingTag = dataService.findOne(TagMetaData.ENTITY_NAME,
  						tag.getString(TagMetaData.IDENTIFIER));
  
  				if (existingTag == null)
  				{
  					dataService.add(TagMetaData.ENTITY_NAME, transformed);
  				}
  				else
  				{
  					dataService.update(TagMetaData.ENTITY_NAME, transformed);
  				}
  			}
  		}
  	}
  
  	/**
  	 * Drops entities and added attributes and reindexes the entities whose attributes were modified.
  	 */
  	@RunAsSystem
  	public void rollbackSchemaChanges(EmxImportJob job)
  	{
  		LOG.info("Rolling back changes.");
  		dataService.delete(LanguageMetaData.ENTITY_NAME, job.metaDataChanges.getAddedLanguages().stream());
  		dropAddedEntities(job.metaDataChanges.getAddedEntities());
  		List<String> entities = dropAddedAttributes(job.metaDataChanges.getAddedAttributes());
  
  		// FIXME import is not transactional, but uses corrective measures to rollback
  		// Reindex
  		Set<String> entitiesToIndex = Sets.newLinkedHashSet(job.source.getEntityNames());
  		entitiesToIndex.addAll(entities);
  		entitiesToIndex.add("tags");
  		entitiesToIndex.add("packages");
  		entitiesToIndex.add("entities");
  		entitiesToIndex.add("attributes");
  
  		reindex(entitiesToIndex);
  	}
  
  	/**
  	 * Reindexes entities
  	 * 
  	 * @param entitiesToIndex
  	 *            Set of entity names
  	 */
  	private void reindex(Set<String> entitiesToIndex)
  	{
  		for (String entity : entitiesToIndex)
  		{
  			if (dataService.hasRepository(entity))
  			{
  				Repository repo = dataService.getRepository(entity);
  				if (repo.getCapabilities().contains(RepositoryCapability.INDEXABLE))
  				{
  					repo.rebuildIndex();
  				}
  			}
  		}
  	}
  
  	/**
  	 * Drops attributes from entities
  	 */
  	private List<String> dropAddedAttributes(ImmutableMap<String, Collection<AttributeMetaData>> addedAttributes)
  	{
  		List<String> entities = Lists.newArrayList(addedAttributes.keySet());
  		Collections.reverse(entities);
  
  		for (String entityName : entities)
  		{
  			for (AttributeMetaData attribute : addedAttributes.get(entityName))
  			{
  				dataService.getMeta().deleteAttribute(entityName, attribute.getName());
  			}
  		}
  		return entities;
  	}
  
  	/**
  	 * Drops added entities in the reverse order in which they were created.
  	 */
  	private void dropAddedEntities(List<String> addedEntities)
  	{
  		// Rollback metadata, create table statements cannot be rolled back, we have to do it ourselves
  		Lists.reverse(addedEntities).forEach(entity -> {
  			try
  			{
  				dataService.getMeta().deleteEntityMeta(entity);
  			}
  			catch (Exception ex)
  			{
  				LOG.error("Failed to rollback creation of entity {}", entity);
  			}
  		});
  	}
  
  	/**
  	 * Updates a repository with entities.
  	 * 
  	 * @param repo
  	 *            the {@link Repository} to update
  	 * @param entities
  	 *            the entities to
  	 * @param dbAction
  	 *            {@link DatabaseAction} describing how to merge the existing entities
  	 * @return number of updated entities
  	 */
  	public int update(Repository repo, Iterable<? extends Entity> entities, DatabaseAction dbAction)
  	{
  		if (entities == null) return 0;
  
  		if (!molgenisPermissionService.hasPermissionOnEntity(repo.getName(), Permission.WRITE))
  		{
  			throw new MolgenisDataAccessException("No WRITE permission on entity '" + repo.getName()
  					+ "'. Is this entity already imported by another user who did not grant you WRITE permission?");
  		}
  
  		String idAttributeName = repo.getEntityMetaData().getIdAttribute().getName();
  		FieldType idDataType = repo.getEntityMetaData().getIdAttribute().getDataType();
  
  		HugeSet<Object> existingIds = new HugeSet<Object>();
  		HugeSet<Object> ids = new HugeSet<Object>();
  		try
  		{
  			for (Entity entity : entities)
  			{
  				Object id = entity.get(idAttributeName);
  