text:
import calendar
import json
import logging
import os
import urllib
import uuid

try:
  from collections import OrderedDict
except ImportError:
  from ordereddict import OrderedDict # Python 2.6

from itertools import chain

from django.contrib.auth import models as auth_models
from django.contrib.contenttypes import generic
from django.contrib.contenttypes.models import ContentType
from django.contrib.staticfiles.storage import staticfiles_storage
from django.core.urlresolvers import reverse, NoReverseMatch
from django.db import connection, models, transaction
from django.db.models import Q
from django.db.models.query import QuerySet
from django.utils.translation import ugettext as _, ugettext_lazy as _t

from settings import HUE_DESKTOP_VERSION

from aws.conf import is_enabled as is_s3_enabled, has_s3_access
from azure.conf import is_adls_enabled, has_adls_access
from dashboard.conf import get_engines
from notebook.conf import SHOW_NOTEBOOKS, get_ordered_interpreters

from desktop import appmanager
from desktop.conf import get_clusters
from desktop.lib.i18n import force_unicode
from desktop.lib.exceptions_renderable import PopupException
from desktop.lib.paths import get_run_root
from desktop.redaction import global_redaction_engine
from desktop.settings import DOCUMENT2_SEARCH_MAX_LENGTH

LOG = logging.getLogger(__name__)

SAMPLE_USER_ID = 1100713
SAMPLE_USER_INSTALL = 'hue'
SAMPLE_USER_OWNERS = ['hue', 'sample']

UTC_TIME_FORMAT = "%Y-%m-%dT%H:%MZ"
HUE_VERSION = None

USER_PREFERENCE_CLUSTER = 'cluster'

def uuid_default():
  return str(uuid.uuid4())

def hue_version():
  global HUE_VERSION

  if HUE_VERSION is None:
    p = get_run_root('cloudera', 'cdh_version.properties')
    if os.path.exists(p):
      HUE_VERSION = _version_from_properties(open(p))
    else:
      HUE_VERSION = HUE_DESKTOP_VERSION

  return HUE_VERSION

def _version_from_properties(f):
  return dict(line.strip().split('=') for line in f.readlines() if len(line.strip().split('=')) == 2).get('version', HUE_DESKTOP_VERSION)

###################################################################################################
# Custom Settings
###################################################################################################

PREFERENCE_IS_WELCOME_TOUR_SEEN = 'is_welcome_tour_seen'

class UserPreferences(models.Model):
  """Holds arbitrary key/value strings."""
  user = models.ForeignKey(auth_models.User)
  key = models.CharField(max_length=20)
  value = models.TextField(max_length=4096)

class Settings(models.Model):
  collect_usage = models.BooleanField(db_index=True, default=True)
  tours_and_tutorials = models.BooleanField(db_index=True, default=True)

  @classmethod
  def get_settings(cls):
    settings, created = Settings.objects.get_or_create(id=1)
    return settings

class DefaultConfigurationManager(models.Manager):

  def get_configuration_for_user(self, app, user):
    """
    :param app: app name
    :param user: User object
    :return: DefaultConfiguration for user, or first group found, or default for app, or None
    """
    try:
      return super(DefaultConfigurationManager, self).get(app=app, user=user)
    except DefaultConfiguration.DoesNotExist:
      pass

    configs = super(DefaultConfigurationManager, self).get_queryset().filter(app=app, groups__in=user.groups.all())
    if configs.count() > 0:
      return configs[0]

    try:
      return super(DefaultConfigurationManager, self).get(app=app, is_default=True)
    except DefaultConfiguration.DoesNotExist:
      pass

    return None

class DefaultConfiguration(models.Model):
  """
  Default values for configuration properties for a given app/editor
  Can be designated as default for all users by is_default flag, or for a specific group or user
  """
  app = models.CharField(max_length=32, null=False, db_index=True, help_text=_t('App that this configuration belongs to.'))
  properties = models.TextField(default='[]', help_text=_t('JSON-formatted default properties values.'))

  is_default = models.BooleanField(default=False, db_index=True)
  groups = models.ManyToManyField(auth_models.Group, db_index=True, db_table='defaultconfiguration_groups')
  user = models.ForeignKey(auth_models.User, blank=True, null=True, db_index=True)

  objects = DefaultConfigurationManager()

  class Meta:
    ordering = ["app", "-is_default", "user"]

  @property
  def properties_list(self):
    """
    :return: Deserialized properties as a list of property objects
    """
    if not self.properties:
      self.properties = json.dumps([])
    return json.loads(self.properties)

  @property
  def properties_dict(self):
    """
    :return: Deserialized properties as a dict of key: value pairs
    """
    if not self.properties:
      self.properties = json.dumps([])
    return dict((prop['key'], prop['value']) for prop in json.loads(self.properties))

  def to_dict(self):
    return {
      'id': self.id,
      'app': self.app,
      'properties': self.properties_list,
      'is_default': self.is_default,
      'group_ids': [group.id for group in self.groups.all()],
      'user': self.user.username if self.user else None
    }

###################################################################################################
# Document1
###################################################################################################
class DocumentTagManager(models.Manager):

  def get_tags(self, user):
    return self.filter(owner=user).distinct()

  def create_tag(self, owner, tag_name):
    if tag_name in DocumentTag.RESERVED:
      raise Exception(_("Can't add %s: it is a reserved tag.") % tag_name)
    else:
      tag, created = DocumentTag.objects.get_or_create(tag=tag_name, owner=owner)
      return tag

  def _get_tag(self, user, name):
    tag, created = DocumentTag.objects.get_or_create(owner=user, tag=name)
    return tag

  def get_default_tag(self, user):
    return self._get_tag(user, DocumentTag.DEFAULT)

  def get_trash_tag(self, user):
    return self._get_tag(user, DocumentTag.TRASH)

  def get_history_tag(self, user):
    return self._get_tag(user, DocumentTag.HISTORY)

  def get_example_tag(self, user):
    return self._get_tag(user, DocumentTag.EXAMPLE)

  def get_imported2_tag(self, user):
    return self._get_tag(user, DocumentTag.IMPORTED2)

  def tag(self, owner, doc_id, tag_name='', tag_id=None):
    try:
      tag = DocumentTag.objects.get(id=tag_id, owner=owner)
      if tag.tag in DocumentTag.RESERVED:
        raise Exception(_("Can't add %s: it is a reserved tag.") % tag)
    except DocumentTag.DoesNotExist:
      tag = self._get_tag(user=owner, name=tag_name)

    doc = Document.objects.get_doc_for_writing(doc_id, owner)
    doc.add_tag(tag)

    return tag

  def untag(self, tag_id, owner, doc_id):
    tag = DocumentTag.objects.get(id=tag_id, owner=owner)

    if tag.tag in DocumentTag.RESERVED:
      raise Exception(_("Can't remove %s: it is a reserved tag.") % tag)

    doc = Document.objects.get_doc_for_writing(doc_id, owner=owner)
    doc.remove_tag(tag)

  def delete_tag(self, tag_id, owner):
    tag = DocumentTag.objects.get(id=tag_id, owner=owner)
    default_tag = DocumentTag.objects.get_default_tag(owner)

    if tag.tag in DocumentTag.RESERVED:
      raise Exception(_("Can't remove %s: it is a reserved tag.") % tag)
    else:
      tag.delete()

    for doc in Document.objects.get_docs(owner).filter(tags=None):
      doc.add_tag(default_tag)

  def update_tags(self, owner, doc_id, tag_ids):
    doc = Document.objects.get_doc_for_writing(doc_id, owner)

    for tag in doc.tags.all():
      if tag.tag not in DocumentTag.RESERVED:
        doc.remove_tag(tag)

    for tag_id in tag_ids:
      tag = DocumentTag.objects.get(id=tag_id, owner=owner)
      if tag.tag not in DocumentTag.RESERVED:
        doc.add_tag(tag)

    return doc

class DocumentTag(models.Model):
  """
  Reserved tags can't be manually removed by the user.
  """
  owner = models.ForeignKey(auth_models.User, db_index=True)
  tag = models.SlugField()

  DEFAULT = 'default' # Always there
  TRASH = 'trash' # There when the document is trashed
  HISTORY = 'history' # There when the document is a submission history
  EXAMPLE = 'example' # Hue examples
  IMPORTED2 = 'imported2' # Was imported to document2

  RESERVED = (DEFAULT, TRASH, HISTORY, EXAMPLE, IMPORTED2)

  objects = DocumentTagManager()

  class Meta:
    unique_together = ('owner', 'tag')

  def __unicode__(self):
    return force_unicode('%s') % (self.tag,)

class DocumentManager(models.Manager):

  def documents(self, user):
    return Document.objects.filter(
        Q(owner=user) |
        Q(documentpermission__users=user) |
        Q(documentpermission__groups__in=user.groups.all())
    ).defer('description', 'extra').distinct()

  def get_docs(self, user, model_class=None, extra=None, qfilter=None):
    docs = Document.objects.documents(user).exclude(name='pig-app-hue-script')

    if model_class is not None:
      ct = ContentType.objects.get_for_model(model_class)
      docs = docs.filter(content_type=ct)

    if extra is not None:
      docs = docs.filter(extra=extra)

    if qfilter is not None:
      docs = docs.filter(qfilter)

    return docs

  def get_doc_for_writing(self, doc_id, user):
    """Fetch a document and confirm that this user can write to it."""
    doc = Document.objects.documents(user).get(id=doc_id)
    doc.can_write_or_exception(user)
    return doc

  def trashed_docs(self, model_class, user):
    tag = DocumentTag.objects.get_trash_tag(user=user)

    return Document.objects.get_docs(user, model_class).filter(tags__in=[tag]).order_by('-last_modified')

  def trashed(self, model_class, user):
    docs = self.trashed_docs(model_class, user)

    return [job.content_object for job in docs if job.content_object]

  def available_docs(self, model_class, user, with_history=False):
    exclude = [DocumentTag.objects.get_trash_tag(user=user)]
    if not with_history:
      exclude.append(DocumentTag.objects.get_history_tag(user=user))

    return Document.objects.get_docs(user, model_class).exclude(tags__in=exclude).order_by('-last_modified')

  def history_docs(self, model_class, user):
    include = [DocumentTag.objects.get_history_tag(user=user)]
    exclude = [DocumentTag.objects.get_trash_tag(user=user)]

    return Document.objects.get_docs(user, model_class).filter(tags__in=include).exclude(tags__in=exclude).order_by('-last_modified')

  def available(self, model_class, user, with_history=False):
    docs = self.available_docs(model_class, user, with_history)

    return [doc.content_object for doc in docs if doc.content_object]

  def can_read_or_exception(self, user, doc_class, doc_id, exception_class=PopupException):
    if doc_id is None:
      return
    try:
      ct = ContentType.objects.get_for_model(doc_class)
      doc = Document.objects.get(object_id=doc_id, content_type=ct)
      if doc.can_read(user):
        return doc
      else:
        message = _("Permission denied. %(username)s does not have the permissions required to access document %(id)s") % \
            {'username': user.username, 'id': doc.id}
        raise exception_class(message)

    except Document.DoesNotExist:
      raise exception_class(_('Document %(id)s does not exist') % {'id': doc_id})

  def can_read(self, user, doc_class, doc_id):
    ct = ContentType.objects.get_for_model(doc_class)
    doc = Document.objects.get(object_id=doc_id, content_type=ct)

    return doc.can_read(user)

  def link(self, content_object, owner, name='', description='', extra=''):
    if not content_object.doc.exists():
      doc = Document.objects.create(
                content_object=content_object,
                owner=owner,
                name=name,
                description=description,
                extra=extra
            )

      tag = DocumentTag.objects.get_default_tag(user=owner)
      doc.tags.add(tag)
      return doc
    else:
      LOG.warn('Object %s already has documents: %s' % (content_object, content_object.doc.all()))
      return content_object.doc.all()[0]

  def sync(self, doc2_only=True):

    def find_jobs_with_no_doc(model):
      jobs = model.objects.filter(doc__isnull=True)
      if model == Document2:
        jobs = jobs.exclude(is_history=True)
      return jobs.select_related('owner')

    def find_oozie_jobs_with_no_doc(model):
      return model.objects.filter(doc__isnull=True).exclude(name__exact='').select_related('owner')

    table_names = connection.introspection.table_names()

    try:
      from oozie.models import Workflow, Coordinator, Bundle

      if \
          Workflow._meta.db_table in table_names or \
          Coordinator._meta.db_table in table_names or \
          Bundle._meta.db_table in table_names:
        with transaction.atomic():
          for job in chain(
              find_oozie_jobs_with_no_doc(Workflow),
              find_oozie_jobs_with_no_doc(Coordinator),
              find_oozie_jobs_with_no_doc(Bundle)):
            doc = Document.objects.link(job, owner=job.owner, name=job.name, description=job.description)

            if job.is_trashed:
              doc.send_to_trash()

            if job.is_shared:
              doc.share_to_default()

            if hasattr(job, 'managed'):
              if not job.managed:
                doc.extra = 'jobsub'
                doc.save()
    except Exception, e:
      LOG.exception('error syncing oozie')

    try:
      from beeswax.models import SavedQuery

      if SavedQuery._meta.db_table in table_names:
        with transaction.atomic():
          for job in find_jobs_with_no_doc(SavedQuery):
            doc = Document.objects.link(job, owner=job.owner, name=job.name, description=job.desc, extra=job.type)
            if job.is_trashed:
              doc.send_to_trash()
    except Exception, e:
      LOG.exception('error syncing beeswax')

    try:
      from pig.models import PigScript

      if PigScript._meta.db_table in table_names:
        with transaction.atomic():
          for job in find_jobs_with_no_doc(PigScript):
            Document.objects.link(job, owner=job.owner, name=job.dict['name'], description='')
    except Exception, e:
      LOG.exception('error syncing pig')

    try:
      from search.models import Collection

      if Collection._meta.db_table in table_names:
        with transaction.atomic():
          for dashboard in Collection.objects.all():
            if 'collection' in dashboard.properties_dict:
              col_dict = dashboard.properties_dict['collection']
              if not 'uuid' in col_dict:
                _uuid = str(uuid.uuid4())
                col_dict['uuid'] = _uuid
                dashboard.update_properties({'collection': col_dict})
                if dashboard.owner is None:
                  from useradmin.models import install_sample_user
                  owner = install_sample_user()
                else:
                  owner = dashboard.owner
                dashboard_doc = Document2.objects.create(name=dashboard.label, uuid=_uuid, type='search-dashboard', owner=owner, description=dashboard.label, data=dashboard.properties)
                Document.objects.link(dashboard_doc, owner=owner, name=dashboard.label, description=dashboard.label, extra='search-dashboard')
                dashboard.save()
    except Exception, e:
      LOG.exception('error syncing search')

    try:
      if Document2._meta.db_table in table_names:
        with transaction.atomic():
          for job in find_jobs_with_no_doc(Document2):
            if job.type == 'oozie-workflow2':
              extra = 'workflow2'
            elif job.type == 'oozie-coordinator2':
              extra = 'coordinator2'
            elif job.type == 'oozie-bundle2':
              extra = 'bundle2'
            elif job.type == 'notebook':
              extra = 'notebook'
            elif job.type == 'search-dashboard':
              extra = 'search-dashboard'
            else:
              extra = ''
            doc = Document.objects.link(job, owner=job.owner, name=job.name, description=job.description, extra=extra)
    except Exception, e:
      LOG.exception('error syncing Document2')

    if not doc2_only and Document._meta.db_table in table_names:
      # Make sure doc have at least a tag
      try:
        for doc in Document.objects.filter(tags=None):
          default_tag = DocumentTag.objects.get_default_tag(doc.owner)
          doc.tags.add(default_tag)
      except Exception, e:
        LOG.exception('error adding at least one tag to docs')

      # Make sure all the sample user documents are shared.
      try:
        with transaction.atomic():
          for doc in Document.objects.filter(owner__username__in=SAMPLE_USER_OWNERS):
            doc.share_to_default()

            tag = DocumentTag.objects.get_example_tag(user=doc.owner)
            doc.tags.add(tag)
            doc_last_modified = doc.last_modified

            doc.save()
            Document.objects.filter(id=doc.id).update(last_modified=doc_last_modified)
      except Exception, e:
        LOG.exception('error sharing sample user documents')

      # For now remove the default tag from the examples
      try:
        for doc in Document.objects.filter(tags__tag=DocumentTag.EXAMPLE):
          default_tag = DocumentTag.objects.get_default_tag(doc.owner)
          doc.tags.remove(default_tag)
      except Exception, e:
        LOG.exception('error removing default tags')

      # ------------------------------------------------------------------------

      LOG.info('Looking for documents that have no object')

      # Delete documents with no object.
      try:
        with transaction.atomic():
          # First, delete all the documents that don't have a content type
          docs = Document.objects.filter(content_type=None)

          if docs:
            LOG.info('Deleting %s doc(s) that do not have a content type' % docs.count())
            docs.delete()

          # Next, it's possible that there are documents pointing at a non-existing
          # content_type. We need to do a left join to find these records, but we
          # can't do this directly in django. To get around writing wrap sql (which
          # might not be portable), we'll use an aggregate to count up all the
          # associated content_types, and delete the documents that have a count of
          # zero.
          #
          # Note we're counting `content_type__name` to force the join.
          docs = Document.objects \
              .values('id') \
              .annotate(content_type_count=models.Count('content_type__name')) \
              .filter(content_type_count=0)

          if docs:
            LOG.info('Deleting %s doc(s) that have invalid content types' % docs.count())
            docs.delete()

          # Finally we need to delete documents with no associated content object.
          # This is tricky because of our use of generic foreign keys. So to do
          # this a bit more efficiently, we'll start with a query of all the
          # documents, then step through each content type and and filter out all
          # the documents it's referencing from our document query. Messy, but it
          # works.

          # TODO: This can be several 100k entries for large databases.
          # Need to figure out a better way to handle this scenario.
          docs = Document.objects.all()

          for content_type in ContentType.objects.all():
            model_class = content_type.model_class()

            # Ignore any types that don't have a model.
            if model_class is None:
              continue

            # Ignore types that don't have a table yet.
            if model_class._meta.db_table not in table_names:
              continue

            # Ignore classes that don't have a 'doc'.
            if not hasattr(model_class, 'doc'):
              continue

            # First create a query that grabs all the document ids for this type.
            docs_from_content = model_class.objects.values('doc__id')

            # Next, filter these from our document query.
            docs = docs.exclude(id__in=docs_from_content)

          if docs.exists():
            LOG.info('Deleting %s documents' % docs.count())
            docs.delete()
      except Exception, e:
        LOG.exception('Error in sync while attempting to delete documents with no object: %s' % e)

class Document(models.Model):
  owner = models.ForeignKey(auth_models.User, db_index=True, verbose_name=_t('Owner'), help_text=_t('User who can own the job.'), related_name='doc_owner')
  name = models.CharField(default='', max_length=255)
  description = models.TextField(default='')

  last_modified = models.DateTimeField(auto_now=True, db_index=True, verbose_name=_t('Last modified'))
  version = models.SmallIntegerField(default=1, verbose_name=_t('Schema version'))
  extra = models.TextField(default='')

  tags = models.ManyToManyField(DocumentTag, db_index=True)

  content_type = models.ForeignKey(ContentType)
  object_id = models.PositiveIntegerField()
  content_object = generic.GenericForeignKey('content_type', 'object_id')

  objects = DocumentManager()

  class Meta:
    unique_together = ('content_type', 'object_id')

  def __unicode__(self):
    return force_unicode('%s %s %s') % (self.content_type, self.name, self.owner)

  def is_editable(self, user):
    """Deprecated by can_read"""
    return self.can_write(user)

  def can_edit_or_exception(self, user, exception_class=PopupException):
    """Deprecated by can_write_or_exception"""
    return self.can_write_or_exception(user, exception_class)

  def add_tag(self, tag):
    self.tags.add(tag)

  def remove_tag(self, tag):
    self.tags.remove(tag)

  def is_trashed(self):
    return DocumentTag.objects.get_trash_tag(user=self.owner) in self.tags.all()

  def is_historic(self):
    return DocumentTag.objects.get_history_tag(user=self.owner) in self.tags.all()

  def send_to_trash(self):
    tag = DocumentTag.objects.get_trash_tag(user=self.owner)
    self.tags.add(tag)

  def restore_from_trash(self):
    tag = DocumentTag.objects.get_trash_tag(user=self.owner)
    self.tags.remove(tag)

  def add_to_history(self):
    tag = DocumentTag.objects.get_history_tag(user=self.owner)
    self.tags.add(tag)

  def remove_from_history(self):
    tag = DocumentTag.objects.get_history_tag(user=self.owner)
    self.tags.remove(tag)

  def share_to_default(self, name='read'):
    DocumentPermission.objects.share_to_default(self, name=name)

  def can_read(self, user):
    return user.is_superuser or self.owner == user or Document.objects.get_docs(user).filter(id=self.id).exists()

  def can_write(self, user):
    perm = self.list_permissions('write')
    return user.is_superuser or self.owner == user or perm.groups.filter(id__in=user.groups.all()).exists() or user in perm.users.all()

  def can_read_or_exception(self, user, exception_class=PopupException):
    if self.can_read(user):
      return True
    else:
      raise exception_class(_("Document does not exist or you don't have the permission to access it."))

  def can_write_or_exception(self, user, exception_class=PopupException):
    if self.can_write(user):
      return True
    else:
      raise exception_class(_("Document does not exist or you don't have the permission to access it."))

  def copy(self, content_object, name, owner, description=None):
    if content_object:
      copy_doc = self

      copy_doc.pk = None
      copy_doc.id = None
      copy_doc.name = name
      copy_doc.owner = owner
      if description:
        copy_doc.description = description

      copy_doc = Document.objects.link(content_object,
                                       owner=copy_doc.owner,
                                       name=copy_doc.name,
                                       description=copy_doc.description,
                                       extra=copy_doc.extra)

      # Update reverse Document relation to new copy
      if content_object.doc.get():
        content_object.doc.get().delete()
      content_object.doc.add(copy_doc)

      return copy_doc
    else:
      raise PopupException(_("Document copy method requires a content_object argument."))

  @property
  def icon(self):
    apps = appmanager.get_apps_dict()

    try:
      if self.extra == 'workflow2':
        return staticfiles_storage.url('oozie/art/icon_oozie_workflow_48.png')
      elif self.extra == 'coordinator2':
        return staticfiles_storage.url('oozie/art/icon_oozie_coordinator_48.png')
      elif self.extra == 'bundle2':
        return staticfiles_storage.url('oozie/art/icon_oozie_bundle_48.png')
      elif self.extra == 'notebook':
        return staticfiles_storage.url('notebook/art/icon_notebook_48.png')
      elif self.extra.startswith('query'):
        if self.extra == 'query-impala':
          return staticfiles_storage.url(apps['impala'].icon_path)
        else:
          return staticfiles_storage.url(apps['beeswax'].icon_path)
      elif self.extra.startswith('search'):
        return staticfiles_storage.url('search/art/icon_search_48.png')
      elif self.content_type.app_label == 'beeswax':
        if self.extra == '0':
          return staticfiles_storage.url(apps['beeswax'].icon_path)
        elif self.extra == '3':
          return staticfiles_storage.url(apps['spark'].icon_path)
        else:
          return staticfiles_storage.url(apps['impala'].icon_path)
      elif self.content_type.app_label == 'oozie':
        if self.extra == 'jobsub':
          return staticfiles_storage.url(apps['jobsub'].icon_path)
        else:
          return staticfiles_storage.url(self.content_type.model_class().ICON)
      elif self.content_type.app_label in apps:
        return staticfiles_storage.url(apps[self.content_type.app_label].icon_path)
      else:
        return staticfiles_storage.url('desktop/art/icon_hue_48.png')
    except Exception, e:
      LOG.warn(force_unicode(e))
      return staticfiles_storage.url('desktop/art/icon_hue_48.png')

  def share(self, users, groups, name='read'):
    DocumentPermission.objects.filter(document=self, name=name).update(users=users, groups=groups, add=True)

  def unshare(self, users, groups, name='read'):
    DocumentPermission.objects.filter(document=self, name=name).update(users=users, groups=groups, add=False)

  def sync_permissions(self, perms_dict):
    """
    Set who else or which other group can interact with the document.

    Example of input: {'read': {'user_ids': [1, 2, 3], 'group_ids': [1, 2, 3]}}
    """
    for name, perm in perms_dict.iteritems():
      users = groups = None
      if perm.get('user_ids'):
        users = auth_models.User.objects.in_bulk(perm.get('user_ids'))
      else:
        users = []

      if perm.get('group_ids'):
        groups = auth_models.Group.objects.in_bulk(perm.get('group_ids'))
      else:
        groups = []

      DocumentPermission.objects.sync(document=self, name=name, users=users, groups=groups)

  def list_permissions(self, perm='read'):
    return DocumentPermission.objects.list(document=self, perm=perm)

  def to_dict(self):
    return {
      'owner': self.owner.username,
      'name': self.name,
      'description': self.description,
      'uuid': None, # no uuid == v1
      'id': self.id,
      'doc1_id': self.id,
      'object_id': self.object_id,
      'type': str(self.content_type),
      'last_modified': self.last_modified.strftime(UTC_TIME_FORMAT),
      'last_modified_ts': calendar.timegm(self.last_modified.utctimetuple()),
      'isSelected': False
    }

class DocumentPermissionManager(models.Manager):

  def _check_perm(self, name):
    perms = (DocumentPermission.READ_PERM, DocumentPermission.WRITE_PERM)
    if name not in perms:
      perms_string = ' and '.join(', '.join(perms).rsplit(', ', 1))
      raise PopupException(_('Only %s permissions are supported, not %s.') % (perms_string, name))

  def share_to_default(self, document, name='read'):
    from useradmin.models import get_default_user_group # Remove build dependency

    self._check_perm(name)

    if name == DocumentPermission.WRITE_PERM:
      perm, created = DocumentPermission.objects.get_or_create(doc=document, perms=DocumentPermission.WRITE_PERM)
    else:
      perm, created = DocumentPermission.objects.get_or_create(doc=document, perms=DocumentPermission.READ_PERM)
    default_group = get_default_user_group()

    if default_group:
      perm.groups.add(default_group)

  def update(self, document, name='read', users=None, groups=None, add=True):
    self._check_perm(name)

    perm, created = DocumentPermission.objects.get_or_create(doc=document, perms=name)

    if users is not None:
      if add:
        perm.users.add(*users)
      else:
        perm.users.remove(*users)

    if groups is not None:
      if add:
        perm.groups.add(*groups)
      else:
        perm.groups.remove(*groups)

    if not perm.users and not perm.groups:
      perm.delete()

  def sync(self, document, name='read', users=None, groups=None):
    self._check_perm(name)

    perm, created = DocumentPermission.objects.get_or_create(doc=document, perms=name)

    if users is not None:
      perm.users = []
      perm.users = users
      perm.save()

    if groups is not None:
      perm.groups = []
      perm.groups = groups
      perm.save()

    if not users and not groups:
      perm.delete()

  def list(self, document, perm='read'):
    perm, created = DocumentPermission.objects.get_or_create(doc=document, perms=perm)
    return perm

class DocumentPermission(models.Model):
  READ_PERM = 'read'
  WRITE_PERM = 'write'

  doc = models.ForeignKey(Document)

  users = models.ManyToManyField(auth_models.User, db_index=True, db_table='documentpermission_users')
  groups = models.ManyToManyField(auth_models.Group, db_index=True, db_table='documentpermission_groups')
  perms = models.CharField(default=READ_PERM, max_length=10, choices=( # one perm
    (READ_PERM, 'read'),
    (WRITE_PERM, 'write'),
  ))

  objects = DocumentPermissionManager()

  class Meta:
    unique_together = ('doc', 'perms')

###################################################################################################
# Document2
###################################################################################################
class FilesystemException(Exception):
  pass

class Document2QueryMixin(object):

  def documents(self, user, perms='both', include_history=False, include_trashed=False, include_managed=False):
    """
    Returns all documents that are owned or shared with the user.
    :param perms: both, shared, owned. Defaults to both.
    :param include_history: boolean flag to return history documents. Defaults to False.
    :param include_trashed: boolean flag to return trashed documents. Defaults to True.
    """
    if perms == 'both':
      docs = self.filter(
        Q(owner=user) |
        Q(document2permission__users=user) |
        Q(document2permission__groups__in=user.groups.all())
      )
    elif perms == 'shared':
      docs = self.filter(
        Q(document2permission__users=user) |
        Q(document2permission__groups__in=user.groups.all())
      )
    else:  # only return documents owned by the user
      docs = self.filter(owner=user)

    if not include_history:
      docs = docs.exclude(is_history=True)

    if not include_managed:
      docs = docs.exclude(is_managed=True)

    if not include_trashed:
      docs = docs.exclude(is_trashed=True)

    return docs.defer('description', 'data', 'extra', 'search').distinct().order_by('-last_modified')

  def search_documents(self, types=None, search_text=None, order_by=None):
    """
    Search for documents based on type filters, search_text or order_by and return a queryset of document objects
    :param types: list of Document2 types (e.g. - query-hive, directory, etc)
    :param search_text: text to search on in the name and description fields
    :param order_by: order by field (e.g. -last_modified, type)
    """
    documents = self

    if types and isinstance(types, list) and types[0] != 'all':
      documents = documents.filter(type__in=types)

    if search_text:
      documents = documents.filter(Q(uuid__icontains=search_text) | Q(name__icontains=search_text) | Q(description__icontains=search_text) |
                                   Q(search__icontains=search_text))

    if order_by:  # TODO: Validate that order_by is a valid sort parameter
      documents = documents.order_by(order_by)

    return documents

class Document2QuerySet(QuerySet, Document2QueryMixin):
    pass

class Document2Manager(models.Manager, Document2QueryMixin):

  def get_query_set(self):
    return Document2QuerySet(self.model, using=self._db)

  # TODO prevent get() in favor of this
  def document(self, user, doc_id):
    return self.documents(user, include_trashed=True, include_history=True).get(id=doc_id)

  def get_by_natural_key(self, uuid, version, is_history):
    return self.get(uuid=uuid, version=version, is_history=is_history)

  def get_by_uuid(self, user, uuid, perm_type='read'):
    """
    Since UUID is not a unique field, but part of a composite unique key, this returns the latest version by UUID
    This should always be used in place of Document2.objects.get(uuid=) when a single document is expected

    :param user: User to check permissions against
    :param uuid
    :param perm_type: permission type to check against
    """
    docs = self.filter(uuid=uuid).order_by('-last_modified')

    if not docs.exists():
      raise FilesystemException(_('Document with UUID %s not found.') % uuid)

    latest_doc = docs[0]

    if perm_type == 'write':
      latest_doc.can_write_or_exception(user)
    else:
      latest_doc.can_read_or_exception(user)

    return latest_doc

  def get_history(self, user, doc_type, include_trashed=False):
    return self.documents(user, perms='owned', include_history=True, include_trashed=include_trashed).filter(type=doc_type, is_history=True)

  def get_tasks_history(self, user):
    return self.documents(user, perms='owned', include_history=True, include_trashed=False, include_managed=True).filter(is_history=True, is_managed=True).exclude(name='pig-app-hue-script').exclude(type='oozie-workflow2')

  def get_home_directory(self, user):
    try:
      return self.get(owner=user, parent_directory=None, name=Document2.HOME_DIR, type='directory')
    except Document2.DoesNotExist:
      return self.create_user_directories(user)
    except Document2.MultipleObjectsReturned:
      LOG.error('Multiple Home directories detected. Merging all into one.')

      home_dirs = list(self.filter(owner=user, parent_directory=None, name=Document2.HOME_DIR, type='directory').order_by('-last_modified'))
      parent_home_dir = home_dirs.pop()
      for dir in home_dirs:
        dir.children.exclude(name='.Trash').update(parent_directory=parent_home_dir)
        dir.delete()

      return parent_home_dir

  def get_by_path(self, user, path):
    """
    This can be an expensive operation b/c we have to traverse the path tree, so if possible, request a document by UUID
    NOTE: get_by_path only works for the owner's documents since it is based off the user's home directory
    """
    cleaned_path = path.rstrip('/')
    doc = Document2.objects.get_home_directory(user)
    if cleaned_path:
      path_tokens = cleaned_path.split('/')[1:]
      for token in path_tokens:
        try:
          doc = doc.children.get(name=token)
        except Document2.DoesNotExist:
          raise FilesystemException(_('Requested invalid path for user %s: %s') % (user.username, path))
        except Document2.MultipleObjectsReturned:
          raise FilesystemException(_('Duplicate documents found for user %s at path: %s') % (user.username, path))

    doc.can_read_or_exception(user)

    return doc

  def create_user_directories(self, user):
    """
    Creates user home and Trash directories if they do not exist and move any orphan documents to home directory
    :param user: User object
    """
    # Edge-case if the user has a legacy home directory with path-name
    Directory.objects.filter(name='/', owner=user).update(name=Document2.HOME_DIR)

    # Get or create home and Trash directories for all users
    home_dir, created = Directory.objects.get_or_create(name=Document2.HOME_DIR, owner=user)

    if created:
      LOG.info('Successfully created home directory for user: %s' % user.username)

    trash_dir, created = Directory.objects.get_or_create(name=Document2.TRASH_DIR, owner=user, parent_directory=home_dir)

    if created:
      LOG.info('Successfully created trash directory for user: %s' % user.username)

    # For any directories or documents that do not have a parent directory, assign it to home directory
    count = 0
    for doc in Document2.objects.filter(owner=user).filter(parent_directory=None).exclude(id__in=[home_dir.id, trash_dir.id]):
      doc.parent_directory = home_dir
      doc.save()
      count += 1

    LOG.info("Moved %d documents to home directory for user: %s" % (count, user.username))
    return home_dir

class Document2(models.Model):

  HOME_DIR = ''
  TRASH_DIR = '.Trash'
  EXAMPLES_DIR = 'examples'

  owner = models.ForeignKey(auth_models.User, db_index=True, verbose_name=_t('Owner'), help_text=_t('Creator.'), related_name='doc2_owner')
  name = models.CharField(default='', max_length=255)
  description = models.TextField(default='')
  uuid = models.CharField(default=uuid_default, max_length=36, db_index=True)
  type = models.CharField(default='', max_length=32, db_index=True, help_text=_t('Type of document, e.g. Hive query, Oozie workflow, Search Dashboard...'))

  data = models.TextField(default='{}')
  extra = models.TextField(default='')
  search = models.TextField(blank=True, null=True, help_text=_t('Searchable text for the document.'))
  # settings = models.TextField(default='{}') # Owner settings like, can other reshare, can change access

  last_modified = models.DateTimeField(auto_now=True, db_index=True, verbose_name=_t('Time last modified'))
  version = models.SmallIntegerField(default=1, verbose_name=_t('Document version'), db_index=True)
  is_history = models.BooleanField(default=False, db_index=True)
  is_managed = models.BooleanField(default=False, db_index=True, verbose_name=_t('If managed under the cover by Hue and never by the user')) # Aka isTask
  is_trashed = models.NullBooleanField(default=False, db_index=True, verbose_name=_t('True if trashed'))

  dependencies = models.ManyToManyField('self', symmetrical=False, related_name='dependents', db_index=True)

  parent_directory = models.ForeignKey('self', blank=True, null=True, related_name='children', on_delete=models.CASCADE)

  doc = generic.GenericRelation(Document, related_name='doc_doc') # Compatibility with Hue 3

  objects = Document2Manager()

  class Meta:
    unique_together = ('uuid', 'version', 'is_history')
    ordering = ["-last_modified", "name"]

  def __unicode__(self):
    res = '%s - %s - %s - %s' % (force_unicode(self.name), self.owner, self.type, self.uuid)
    return force_unicode(res)

  @property
  def data_dict(self):
    if not self.data:
      self.data = json.dumps({})
    data_python = json.loads(self.data)
    return data_python

  @property
  def path(self):
    quoted_name = urllib.quote(self.name.encode('utf-8'))
    if self.parent_directory:
      return '%s/%s' % (self.parent_directory.path, quoted_name)
    else:
      return quoted_name

  @property
  def dirname(self):
    return os.path.dirname(self.path) or '/'

  @property
  def is_directory(self):
    return self.type == 'directory'

  @property
  def is_home_directory(self):
    return self.is_directory and self.parent_directory == None and self.name == self.HOME_DIR

  @property
  def is_trash_directory(self):
    return self.is_directory and self.name == self.TRASH_DIR

  def natural_key(self):
    return (self.uuid, self.version, self.is_history)

  def copy(self, name, owner, description=None):
    copy_doc = self

    copy_doc.pk = None
    copy_doc.id = None
    copy_doc.uuid = uuid_default()
    copy_doc.name = name
    copy_doc.owner = owner
    if description:
      copy_doc.description = description
    copy_doc.save()
    return copy_doc

  def update_data(self, post_data):
    data_dict = self.data_dict
    data_dict.update(post_data)
    self.data = json.dumps(data_dict)

  def get_absolute_url(self):
    url = None
    try:
      if self.type == 'oozie-coordinator2':
        url = reverse('oozie:edit_coordinator') + '?coordinator=' + str(self.id)
      elif self.type == 'oozie-bundle2':
        url = reverse('oozie:edit_bundle') + '?bundle=' + str(self.id)
      elif self.type.startswith('query'):
        url = '/editor' + '?editor=' + str(self.id)
      elif self.type == 'directory':
        url = '/home2' + '?uuid=' + self.uuid
      elif self.type == 'notebook':
        url = reverse('notebook:notebook') + '?notebook=' + str(self.id)
      elif self.type == 'search-dashboard':
        url = reverse('search:index') + '?collection=' + str(self.id)
      elif self.type == 'link-pigscript':
        url = reverse('pig:index') + '#edit/%s' % self.data_dict.get('object_id', '')
      elif self.type == 'link-workflow':
        url = '/jobsub/#edit-design/%s' % self.data_dict.get('object_id', '')
      else:
        url = reverse('oozie:edit_workflow') + '?workflow=' + str(self.id)
    except NoReverseMatch:
      LOG.warn('Could not perform reverse lookup for type %s, app may be blacklisted.' % self.type)
    return url

  def to_dict(self):
    return {
      'owner': self.owner.username,
      'name': self.name,
      'path': self.path or '/',
      'description': self.description,
      'uuid': self.uuid,
      'id': self.id,
      'doc1_id': self.doc.get().id if self.doc.exists() else -1,
      'parent_uuid': self.parent_directory.uuid if self.parent_directory else None,
      'type': self.type,
      'perms': self._massage_permissions(),
      'last_modified': self.last_modified.strftime(UTC_TIME_FORMAT),
      'last_modified_ts': calendar.timegm(self.last_modified.utctimetuple()),
      'is_managed': self.is_managed,
      'isSelected': False,
      'absoluteUrl': self.get_absolute_url(),
    }

  def get_history(self):
    return self.dependencies.filter(is_history=True).order_by('-last_modified')

  def add_to_history(self, user, data_dict):
    doc_id = self.id # Need to copy as the clone messes it

    history_doc = self.copy(name=self.name, owner=user)
    history_doc.update_data({'history': data_dict})
    history_doc.is_history = True
    history_doc.last_modified = None
    history_doc.save()

    Document2.objects.get(id=doc_id).dependencies.add(history_doc)
    return history_doc

  def save(self, *args, **kwargs):
    # Set document parent to home directory if parent directory isn't specified
    if not self.parent_directory and not self.is_home_directory and not self.is_trash_directory:
      home_dir = Document2.objects.get_home_directory(self.owner)
      self.parent_directory = home_dir

    # Run validations
    self.validate()

    # Redact query if needed
    self._redact_query()

    if self.search:
      self.search = self.search[:DOCUMENT2_SEARCH_MAX_LENGTH]

    super(Document2, self).save(*args, **kwargs)

    # Inherit shared permissions from parent directory, must be done after save b/c new doc needs ID
    self.inherit_permissions()

  def validate(self):
    # Validate home and Trash directories are only created once per user and cannot be created or modified after
    if self.name in [Document2.HOME_DIR, Document2.TRASH_DIR] and self.type == 'directory' and \
          Document2.objects.filter(name=self.name, owner=self.owner, type='directory').exists():
      raise FilesystemException(_('Cannot create or modify directory with name: %s') % self.name)

    # Validate that parent directory does not create cycle
    if self._contains_cycle():
      raise FilesystemException(_('Cannot save document %s under parent directory %s due to circular dependency') %
                                (self.name, self.parent_directory.uuid))

  def inherit_permissions(self):
    if self.parent_directory is not None:
      parent_perms = Document2Permission.objects.filter(doc=self.parent_directory)
      for perm in parent_perms:
        self.share(self.owner, name=perm.perms, users=perm.users.all(), groups=perm.groups.all())

  def move(self, directory, user):
    if not directory.is_directory:
      raise FilesystemException(_('Target with UUID %s is not a directory') % directory.uuid)

    if directory.can_write_or_exception(user=user):
      # Restore last_modified date after save
      original_last_modified = self.last_modified
      self.parent_directory = directory
      self.save()
      # Use update instead of save() so that last_modified date is not modified automatically
      Document2.objects.filter(id=self.id).update(last_modified=original_last_modified)

    return self

  def trash(self):
    try:
      trash_dir = Directory.objects.get(name=self.TRASH_DIR, owner=self.owner)
      self.move(trash_dir, self.owner)
      self.is_trashed = True
      self.save()

      if self.is_directory:
        children_ids = self._get_child_ids_recursively(self.id)
        Document2.objects.filter(id__in=children_ids).update(is_trashed=True)
    except Document2.MultipleObjectsReturned:
      LOG.error('Multiple Trash directories detected. Merging all into one.')

      trash_dirs = list(Directory.objects.filter(name=self.TRASH_DIR, owner=self.owner).order_by('-last_modified'))
      parent_trash_dir = trash_dirs.pop()
      for dir in trash_dirs:
        dir.children.update(parent_directory=parent_trash_dir)
        dir.delete()

      self.move(parent_trash_dir, self.owner)

  def restore(self):
    # Currently restoring any doucment to /home
    home_dir = Document2.objects.get_home_directory(self.owner)
    self.move(home_dir, self.owner)
    self.is_trashed = False
    self.save()

    if self.is_directory:
      children_ids = self._get_child_ids_recursively(self.id)
      Document2.objects.filter(id__in=children_ids).update(is_trashed=False)

  def _get_child_ids_recursively(self, directory_id):
    """
    Returns the list of all children ids for a given directory id recursively, excluding history documents
    """
    directory = Directory.objects.get(id=directory_id)
    children_ids = []
    for child in directory.children.filter(is_history=False).filter(is_managed=False):
      children_ids.append(child.id)
      if child.is_directory:
        children_ids.extend(self._get_child_ids_recursively(child.id))
    return children_ids

  def can_read(self, user):
    perm = self.get_permission('read')
    has_read_permissions = perm.user_has_access(user) if perm else False
    return user.is_superuser or self.owner == user or self.can_write(user) or has_read_permissions

  def can_read_or_exception(self, user):
    if self.can_read(user):
      return True
    else:
      raise PopupException(_("Document does not exist or you don't have the permission to access it."))

  def can_write(self, user):
    perm = self.get_permission('write')
    has_write_permissions = perm.user_has_access(user) if perm else False
    return user.is_superuser or self.owner == user or has_write_permissions

  def can_write_or_exception(self, user):
    if self.can_write(user):
      return True
    else:
      raise PopupException(_("Document does not exist or you don't have the permission to access it."))

  def get_permission(self, perm='read'):
    try:
      return Document2Permission.objects.get(doc=self, perms=perm)
    except Document2Permission.DoesNotExist:
      return None

  def share(self, user, name='read', users=None, groups=None):
    try:
      with transaction.atomic():
        self.update_permission(user, name, users, groups)
        # For directories, update all children recursively with same permissions
        for child in self.children.all():
          child.share(user, name, users, groups)
    except Exception, e:
      raise PopupException(_("Failed to share document: %s") % e)
    return self

  def update_permission(self, user, name='read', users=None, groups=None):
    # Check if user has access to grant permissions
    if users or groups:
      if name == 'read':
        self.can_read_or_exception(user)
      elif name == 'write':
        self.can_write_or_exception(user)
      else:
        raise ValueError(_('Invalid permission type: %s') % name)

    perm, created = Document2Permission.objects.get_or_create(doc=self, perms=name)

    perm.users = []
    if users is not None:
      perm.users = users

    perm.groups = []
    if groups is not None:
      perm.groups = groups

    perm.save()

  def _get_doc1(self, doc2_type=None):
    if not doc2_type:
      doc2_type = self.type

    try:
      doc = self.doc.get()
    except Exception, e:
      LOG.error('Exception when retrieving document object for saved query: %s' % e)
      doc = Document.objects.link(
        self,
        owner=self.owner,
        name=self.name,
        description=self.description,
        extra=doc2_type
      )

    return doc

  def _massage_permissions(self):
    """
    Returns the permissions for a given document as a dictionary
    """
    permissions = {
      'read': {'users': [], 'groups': []},
      'write': {'users': [], 'groups': []}
    }

    read_perms = self.get_permission(perm='read')
    write_perms = self.get_permission(perm='write')

    if read_perms:
      permissions.update(read_perms.to_dict())
    if write_perms:
      permissions.update(write_perms.to_dict())

    return permissions

  def _redact_query(self):
    """
    Optionally mask out the query from being saved to the database. This is because if the database contains sensitive
    information like personally identifiable information, that information could be leaked into the Hue database and
    logfiles.
    """
    if global_redaction_engine.is_enabled() and (self.type == 'notebook' or self.type.startswith('query')):
      data_dict = self.data_dict
      snippets = data_dict.get('snippets', [])
      for snippet in snippets:
        if snippet['type'] in ('hive', 'impala'):  # TODO: Pull SQL types from canonical lookup
          redacted_statement_raw = global_redaction_engine.redact(snippet['statement_raw'])
          if snippet['statement_raw'] != redacted_statement_raw:
            snippet['statement_raw'] = redacted_statement_raw
            snippet['statement'] = global_redaction_engine.redact(snippet['statement'])
            snippet['is_redacted'] = True
      self.data = json.dumps(data_dict)
      self.search = global_redaction_engine.redact(self.search)

  def _contains_cycle(self):
    """
    Uses Floyd's cycle-detection algorithm to detect a cycle (aka Tortoise and Hare)
    https://en.wikipedia.org/wiki/Cycle_detection#Tortoise_and_hare
    """
    # Test base case where self.uuid == self.parent_directory.uuid first
    if self.parent_directory is not None and self.parent_directory.uuid == self.uuid:
      return True

    slow = self
    fast = self
    while True:
      slow = slow.parent_directory
      if slow and slow.uuid == self.uuid:
        slow = self

      if fast.parent_directory is not None:
        if fast.parent_directory.uuid == self.uuid:
          fast = self.parent_directory.parent_directory
        else:
          fast = fast.parent_directory.parent_directory
      else:
        return False

      if slow is None or fast is None:
        return False

      if slow == fast:
        return True

class DirectoryManager(Document2Manager):

  def get_queryset(self):
    return super(DirectoryManager, self).get_queryset().filter(type='directory')

class Directory(Document2):
  # e.g. name = '/' or '/dir1/dir2/f3'

  objects = DirectoryManager()

  class Meta:
    proxy = True

  def get_children_documents(self):
    """
    Returns the children documents for a given directory, excluding history documents
    """
    documents = self.children.filter(is_history=False).filter(is_managed=False)  # TODO: perms
    return documents

  def get_children_and_shared_documents(self, user):
    """
    Returns the children and shared documents for a given directory, excluding history documents
    """
    # Get documents that are direct children, or shared with but not owned by the current user
    documents = Document2.objects.filter(
        Q(parent_directory=self) |
        ( (Q(document2permission__users=user) | Q(document2permission__groups__in=user.groups.all())) &
          ~Q(owner=user) )
      )

    documents = documents.exclude(is_history=True).exclude(is_managed=True)

    # Excluding all trashed docs across users
    documents = documents.exclude(is_trashed=True)

    # Optimizing roll up for /home by checking only with directories instead of all documents
    # For all other directories roll up is done in _filter_documents()
    directories_all = Document2.objects.filter(type='directory').exclude(id=self.id)
    directories_inside_home = directories_all.filter(
      (Q(document2permission__users=user) | Q(document2permission__groups__in=user.groups.all())) &
        ~Q(owner=user)
    )
    documents = documents.exclude(parent_directory__in=directories_inside_home)

    return documents.defer('description', 'data', 'extra', 'search').distinct().order_by('-last_modified')

  def save(self, *args, **kwargs):
    self.type = 'directory'
    super(Directory, self).save(*args, **kwargs)

class Document2Permission(models.Model):
  """
  Combine either:
   - regular perms (listed)
   - link
  """
  READ_PERM = 'read'
  WRITE_PERM = 'write'
  COMMENT_PERM = 'comment'

  doc = models.ForeignKey(Document2)

  users = models.ManyToManyField(auth_models.User, db_index=True, db_table='documentpermission2_users')
  groups = models.ManyToManyField(auth_models.Group, db_index=True, db_table='documentpermission2_groups')

  perms = models.CharField(default=READ_PERM, max_length=10, db_index=True, choices=( # one perm
    (READ_PERM, 'read'),
    (WRITE_PERM, 'write'),
    (COMMENT_PERM, 'comment'), # PLAYER PERM?
  ))

  # link = models.CharField(default=uuid_default, max_length=255, unique=True) # Short link like dropbox
  # embed

  class Meta:
    unique_together = ('doc', 'perms')

  def to_dict(self):
    return {
      self.perms: {
        'users': [{'id': perm_user.id, 'username': perm_user.username} for perm_user in self.users.all()],
        'groups': [{'id': perm_group.id, 'name': perm_group.name} for perm_group in self.groups.all()]
      }
    }

  def user_has_access(self, user):
    """
    Returns true if the given user has permissions based on users, groups, or all flag
    """
    return self.groups.filter(id__in=user.groups.all()).exists() or user in self.users.all()

def get_cluster_config(user):
  cluster_type = Cluster(user).get_type()
  cluster_config = ClusterConfig(user, cluster_type=cluster_type)

  return cluster_config.get_config()

DATAENG = 'dataeng'
IMPALAUI = 'impalaui'

class ClusterConfig():

  def __init__(self, user, apps=None, cluster_type='ini'):
    self.user = user
    self.apps = appmanager.get_apps_dict(self.user) if apps is None else apps
    self.cluster_type = cluster_type

  def refreshConfig(self):
    # TODO: reload "some ini sections"
    pass

  def get_config(self):
    app_config = self.get_apps()
    editors = app_config.get('editor')
    main_button_action = self.get_main_quick_action(app_config)

    if main_button_action.get('is_sql'):
      default_sql_interpreter = main_button_action['type']
    else:
      default_sql_interpreter = editors and editors['default_sql_interpreter']

    return {
      'app_config': app_config,
      'main_button_action': main_button_action,
      'button_actions': [
        app for app in [
          editors,
          app_config.get('dashboard'),
          app_config.get('scheduler')
        ] if app is not None
      ],
      'default_sql_interpreter': default_sql_interpreter,
      'cluster_type': self.cluster_type
    }

  def get_apps(self):
    apps = OrderedDict([app for app in [
      ('editor', self._get_editor()),
      ('dashboard', self._get_dashboard()),
      ('browser', self._get_browser()),
      ('scheduler', self._get_scheduler()),
      ('sdkapps', self._get_sdk_apps()),
      ('home', self._get_home()),
    ] if app[1]])

    return apps

  def get_main_quick_action(self, apps):
    if not apps:
      raise PopupException(_('No permission to any app.'))

    default_app = apps.values()[0]
    default_interpreter = default_app.get('interpreters')

    try:
      user_default_app = json.loads(UserPreferences.objects.get(user=self.user, key='default_app').value)
      if apps.get(user_default_app['app']):
        default_interpreter = []
        default_app = apps[user_default_app['app']]
        if default_app.get('interpreters'):
          interpreters = [interpreter for interpreter in default_app['interpreters'] if interpreter['type'] == user_default_app['interpreter']]
          if interpreters:
            default_interpreter = interpreters
    except UserPreferences.DoesNotExist:
      pass
    except Exception:
      LOG.exception('Could not load back default app')

    if default_interpreter:
      return default_interpreter[0]
    else:
      return default_app

  def _get_home(self):
    return {
      'name': 'home',
      'displayName': _('Home'),
      'buttonName': _('Documents'),
      'interpreters': [],
      'page': '/home'
    }

  def _get_editor(self):
    interpreters = []

    _interpreters = get_ordered_interpreters(self.user)
    if self.cluster_type == DATAENG:
      _interpreters = [interpreter for interpreter in _interpreters if interpreter['type'] in ('hive', 'spark2', 'mapreduce')]
    elif self.cluster_type == IMPALAUI:
      _interpreters = [interpreter for interpreter in _interpreters if interpreter['type'] == 'impala']

    for interpreter in _interpreters:
      interpreters.append({
        'name': interpreter['name'],
        'type': interpreter['type'],
        'displayName': interpreter['name'],
        'buttonName': _('Query'),
        'tooltip': _('%s Query') % interpreter['type'].title(),
        'page': '/editor/?type=%(type)s' % interpreter,
        'is_sql': interpreter['is_sql']
      })

    if SHOW_NOTEBOOKS.get() and (self.cluster_type not in (DATAENG, IMPALAUI)):
      try:
        first_non_sql_index = [interpreter['is_sql'] for interpreter in interpreters].index(False)
      except ValueError:
        first_non_sql_index = 0
      interpreters.insert(first_non_sql_index, {
        'name': 'notebook',
        'type': 'notebook',
        'displayName': 'Notebook',
        'buttonName': _('Notebook'),
        'tooltip': _('Notebook'),
        'page': '/notebook',
        'is_sql': False
      })

    if interpreters:
      return {
        'name': 'editor',
        'displayName': _('Editor'),
        'buttonName': _('Query'),
        'interpreters': interpreters,
        'interpreter_names': [interpreter['type'] for interpreter in interpreters],
        'page': interpreters[0]['page'],
        'default_sql_interpreter': next((interpreter['type'] for interpreter in interpreters if interpreter.get('is_sql')), 'hive')
      }
    else:
      return None

  def _get_dashboard(self):
    interpreters = get_engines(self.user)

    if interpreters and (self.cluster_type not in (DATAENG, IMPALAUI)):
      return {
        'name': 'dashboard',
        'displayName': _('Dashboard'),
        'buttonName': _('Dashboard'),
        'interpreters': [{
            'type': interpreter['type'],
            'displayName': interpreter['type'].title(),
            'buttonName': interpreter['type'].title(),
            'page': '/dashboard/new_search?engine=%(type)s' % interpreter,
            'tooltip': _('%s Dashboard') % interpreter['type'].title()
          } for interpreter in interpreters],
        'page': '/dashboard/new_search?engine=%(type)s' % interpreters[0]
      }
    else:
      return None

  def _get_browser(self):
    interpreters = []

    if 'filebrowser' in self.apps and (self.cluster_type not in (DATAENG, IMPALAUI)):
      interpreters.append({
        'type': 'hdfs',
        'displayName': _('Files'),
        'buttonName': _('Browse'),
        'tooltip': _('Files'),
        'page': '/filebrowser/' + (not self.user.is_anonymous() and 'view=' + self.user.get_home_directory() or '')
      })

    if is_s3_enabled() and has_s3_access(self.user):
      interpreters.append({
        'type': 's3',
        'displayName': _('S3'),
        'buttonName': _('Browse'),
        'tooltip': _('S3'),
        'page': '/filebrowser/view=S3A://'
      })

    if is_adls_enabled() and has_adls_access(self.user):
      interpreters.append({
        'type': 'adls',
        'displayName': _('ADLS'),
        'buttonName': _('Browse'),
        'tooltip': _('ADLS'),
        'page': '/filebrowser/view=adl:/'
      })

    if 'metastore' in self.apps:
      interpreters.append({
        'type': 'tables',
        'displayName': _('Tables'),
        'buttonName': _('Browse'),
        'tooltip': _('Tables'),
        'page': '/metastore/tables'
      })

    if 'search' in self.apps and (self.cluster_type not in (DATAENG, IMPALAUI)):
      interpreters.append({
        'type': 'indexes',
        'displayName': _('Indexes'),
        'buttonName': _('Dashboard'),
        'tooltip': _('Indexes'),
        'page': '/indexer/'
      })

    if 'jobbrowser' in self.apps and self.cluster_type != IMPALAUI:
      if self.cluster_type == DATAENG:
        interpreters.append({
          'type': 'dataeng',
          'displayName': _('Jobs'),
          'buttonName': _('Jobs'),
          'tooltip': _('Jobs'),
          'page': '/jobbrowser/'
        })
      else:
        from hadoop.cluster import get_default_yarncluster # Circular loop
        if get_default_yarncluster():
          interpreters.append({
            'type': 'yarn',
            'displayName': _('Jobs'),
            'buttonName': _('Jobs'),
            'tooltip': _('Jobs'),
            'page': '/jobbrowser/'
          })

    if 'hbase' in self.apps and (self.cluster_type not in (DATAENG, IMPALAUI)):
      interpreters.append({
        'type': 'hbase',
        'displayName': _('HBase'),
        'buttonName': _('Browse'),
        'tooltip': _('HBase'),
        'page': '/hbase/'
      })

    if 'security' in self.apps and (self.cluster_type not in (DATAENG, IMPALAUI)):
      interpreters.append({
        'type': 'security',
        'displayName': _('Security'),
        'buttonName': _('Browse'),
        'tooltip': _('Security'),
        'page': '/security/hive'
      })

    if 'sqoop' in self.apps and (self.cluster_type not in (DATAENG, IMPALAUI)):
      interpreters.append({
        'type': 'sqoop',
        'displayName': _('Sqoop'),
        'buttonName': _('Browse'),
        'tooltip': _('Sqoop'),
        'page': '/sqoop'
      })

    if interpreters:
      return {
          'name': 'browser',
          'displayName': _('Browsers'),
          'buttonName': _('Browse'),
          'interpreters': interpreters,
          'interpreter_names': [interpreter['type'] for interpreter in interpreters],
        }
    else:
      return None

  def _get_scheduler(self):
    interpreters = [{
        'type': 'oozie-workflow',
        'displayName': _('Workflow'),
        'buttonName': _('Schedule'),
        'tooltip': _('Workflow'),
        'page': '/oozie/editor/workflow/new/'
      }, {
        'type': 'oozie-coordinator',
        'displayName': _('Schedule'),
        'buttonName': _('Schedule'),
        'tooltip': _('Schedule'),
        'page': '/oozie/editor/coordinator/new/'
      }, {
        'type': 'oozie-bundle',
        'displayName': _('Bundle'),
        'buttonName': _('Schedule'),
        'tooltip': _('Bundle'),
        'page': '/oozie/editor/bundle/new/'
      }
    ]

    if 'oozie' in self.apps and not (self.user.has_hue_permission(action="disable_editor_access", app="oozie") and not self.user.is_superuser) and self.cluster_type != 'impalaui':
      return {
          'name': 'oozie',
          'displayName': _('Scheduler'),
          'buttonName': _('Schedule'),
          'interpreters': interpreters,
          'page': interpreters[0]['page']
        }
    else:
      return None

  def _get_sdk_apps(self):
    current_app, other_apps, apps_list = _get_apps(self.user)

    interpreters = []

    for other in other_apps:
      interpreters.append({
        'type': other.display_name,
        'displayName': other.nice_name,
        'buttonName': other.nice_name,
        'tooltip': other.nice_name,
        'page': '/%s' % other.nice_name
      })

    if interpreters:
      return {
          'name': 'other',
          'displayName': _('Other Apps'),
          'buttonName': _('Other'),
          'interpreters': interpreters,
        }
    else:
      return None

class Cluster():

  def __init__(self, user):
    self.user = user
    self.default_cluster = get_user_preferences(self.user, key=USER_PREFERENCE_CLUSTER)
    self.data = {}

    if self.default_cluster:
      clusters = get_clusters()
      cluster_name = json.loads(self.default_cluster[USER_PREFERENCE_CLUSTER]).get('name')
      self.data = cluster_name and clusters.get(cluster_name) and clusters[cluster_name] or None

  def get_type(self):
    return self.data and self.data['type'] or 'ini'

  def get_interface(self):
    return json.loads(self.default_cluster[USER_PREFERENCE_CLUSTER]).get('interface')

  def get_id(self):
    return json.loads(self.default_cluster[USER_PREFERENCE_CLUSTER]).get('id')

  def get_list_interface_indexes(self):
    default_cluster_index = 0
    default_cluster_interface = ''

    clusters = get_clusters()
    default_cluster = get_user_preferences(self.user, key=USER_PREFERENCE_CLUSTER)

    if clusters and default_cluster:
      default_cluster_json = json.loads(default_cluster[USER_PREFERENCE_CLUSTER])
      default_cluster_name = default_cluster_json.get('name')

      default_cluster_index = default_cluster_name in clusters.keys() and clusters.keys().index(default_cluster_name) or 0
      default_cluster_interface = default_cluster_json.get('interface', '')

    return default_cluster_index, default_cluster_interface

def _get_apps(user, section=None):
  current_app = None
  other_apps = []
  if user.is_authenticated():
    apps = appmanager.get_apps(user)
    apps_list = appmanager.get_apps_dict(user)
    for app in apps:
      if app.display_name not in [
          'beeswax', 'impala', 'pig', 'jobsub', 'jobbrowser', 'metastore', 'hbase', 'sqoop', 'oozie', 'filebrowser',
          'useradmin', 'search', 'help', 'about', 'zookeeper', 'proxy', 'rdbms', 'spark', 'indexer', 'security', 'notebook'] and app.menu_index != -1:
        other_apps.append(app)
      if section == app.display_name:
        current_app = app
  else:
    apps_list = []

  return current_app, other_apps, apps_list

def get_user_preferences(user, key=None):
  if key is not None:
    try:
      x = UserPreferences.objects.get(user=user, key=key)
      return {key: x.value}
    except UserPreferences.DoesNotExist:
      return None
  else:
    return dict((x.key, x.value) for x in UserPreferences.objects.filter(user=user))

def set_user_preferences(user, key, value):
  try:
    x = UserPreferences.objects.get(user=user, key=key)
  except UserPreferences.DoesNotExist:
    x = UserPreferences(user=user, key=key)
  x.value = value
  x.save()
  return x

def get_data_link(meta):
  link = None

  if not meta.get('type'):
    pass
  elif meta['type'] == 'hbase':
    link = '/hbase/#Cluster/%(table)s/query/%(row_key)s' % meta
    if 'col' in meta:
      link += '[%(fam)s:%(col)s]' % meta
    elif 'fam' in meta:
      link += '[%(fam)s]' % meta
  elif meta['type'] == 'hdfs':
    link = '/filebrowser/view=%(path)s' % meta # Could add a byte #
  elif meta['type'] == 'link':
    link = meta['link']
  elif meta['type'] == 'hive':
    link = '/metastore/table/%(database)s/%(table)s' % meta # Could also add col=val

  return link


text:
<法規>
<法規性質>命令</法規性質>
<英文法規名稱>The Enforcement Rules of the Conclusion of Treaties Act</英文法規名稱>
<中文法規名稱>條約締結法施行細則</中文法規名稱>
<法規網址>https://law.moj.gov.tw/Eng/LawClass/LawAll.aspx?pcode=E0020023</法規網址>
<最新英文異動日期>20151120</最新英文異動日期>
<廢止註記>
</廢止註記>
<附件 />
<沿革內容><![CDATA[1.Promulgated under MOFA's Administrative Order No. 10425530050 on November 20, 2015]]></沿革內容>
<前言><![CDATA[]]></前言>
<法規內容>
  <條文>
    <條號>Article 1</條號>
    <條文內容><![CDATA[The Enforcement Rules are established in accordance with Article 19 of the Conclusion of Treaties Act (Hereinafter, "the Act").]]></條文內容>
  </條文>
  <條文>
    <條號>Article 2</條號>
    <條文內容><![CDATA[The institutions and organizations authorized by the competent administrative authorities of the central government referred to in Article 2 of the Act mean diplomatic missions established in countries that do not have official diplomatic ties with the Republic of China, or institutions or organizations authorized by law to assist international affairs, except for institutions or organizations that are designated by the Ministry of Foreign Affairs (hereinafter referred to as "MOFA") to assist special cases resulting from diplomatic circumstances.
The international organizations referred to in Article 2 of the Act mean international inter-governmental organizations.
The foreign institutions and organizations authorized by their government referred to in Article 2 of the Act mean organizations or representative institutions that are established in the Republic of China by the central government of foreign countries that do not have official diplomatic ties with the Republic of China, to assist diplomatic, consular, cultural, and commercial affairs of the Republic of China, except for institutions or organizations that are designated by MOFA to assist special cases resulting from diplomatic circumstances.]]></條文內容>
  </條文>
  <條文>
    <條號>Article 3</條號>
    <條文內容><![CDATA[If authorities other than MOFA consider a treaty or agreement to be specialized or technical in content, and that the said authorities considers that it is suitable to be in charge of such matters or should work with MOFA, the said authorities shall request MOFA's approval or submit the request to the Executive Yuan for approval pursuant to the proviso of Article 4, Paragraph 1 of the Act.
If MOFA considers a treaty or agreement to be specialized or technical in content and that it may work with other authorities or allow other authorities to be in charge of such matters, MOFA shall contact the authorities or submit the request to the Executive Yuan for approval.]]></條文內容>
  </條文>
  <條文>
    <條號>Article 4</條號>
    <條文內容><![CDATA[Before an agreement is reached regarding the text of a treaty that involves issues of national economy and national interests, the competent authorities, if necessary, may hold public hearings or liaison meetings to explain and communicate with representatives or stakeholders from industries regarding matters that may lead to disputes, affect industries, or arouse public concern, and gather the opinions given by stakeholders, scholars, and experts; the competent authorities shall report the current progress to the Legislative Yuan.
The competent authorities may appoint relevant central or local competent authorities to hold the public hearings or liaison meetings described in the preceding paragraph.]]></條文內容>
  </條文>
  <條文>
    <條號>Article 5</條號>
    <條文內容><![CDATA[The "pressing time constraint" referred to in Article 7 of the Act means special circumstances related to national security, national interest, and the welfare of the people in emergency situations, such as establishing diplomatic relations, establishing offices in foreign countries, humanitarian assistance, severe disaster responses at home and abroad, or disease prevention.
The so-called receiving the approval from the Executive Yuan referred to in Article 7 of the Act means the competent authorities may seek the approval in person or via telephone, text message, communication software, or fax.
If any submissions to the Executive Yuan for approval pursuant to Article 8 and Article 12 of the Act lack a Chinese version, the competent authorities shall submit Chinese translations.]]></條文內容>
  </條文>
  <條文>
    <條號>Article 6</條號>
    <條文內容><![CDATA[When a treaty that involves the national economy and national interests has been signed pursuant to Article 8 of the Act and has been submitted via the Executive Yuan to the Legislative Yuan for deliberations, the competent authorities shall provide the following information:
(1) Substantial proposals and remedies to the facts that affect the assessment.
(2) Budget estimation and sources of funding to implement the treaty.]]></條文內容>
  </條文>
  <條文>
    <條號>Article 7</條號>
    <條文內容><![CDATA[When the Legislative Yuan submits a reservation clause pursuant to Article 10, Paragraph 1 of the Act, the competent authorities shall raise the matters to be reserved during the deposit of the instruments of ratification, acceptance, approval, and accession of the clause unless the treaty has stipulated otherwise.
The matters to be reserved that are prescribed in the preceding paragraph may be included in the instruments of ratification, acceptance, approval, and accession of the matters, or may be submitted via its attachment.
After receiving the resolution of the Legislative Yuan that is made pursuant to Article 10, Paragraph 2 of the Act, the competent authorities shall appoint another date for renegotiation with the signatory depending on the urgency and the necessity. The competent authorities shall explain the progress of the renegotiation to the Legislative Yuan as soon as possible.]]></條文內容>
  </條文>
  <條文>
    <條號>Article 8</條號>
    <條文內容><![CDATA[After a treaty becomes effective, the competent authorities shall send the treaty to the Executive Yuan, which shall submit them to the President for promulgation, and submit the treaty's effective date pursuant to Article 8 and Article 11, Paragraph 1 of the Act.]]></條文內容>
  </條文>
  <條文>
    <條號>Article 9</條號>
    <條文內容><![CDATA[The publishing in a proper way referred to in Article 12 of the Act means publication in the gazette or on the Internet.]]></條文內容>
  </條文>
  <條文>
    <條號>Article 10</條號>
    <條文內容><![CDATA[When submitting the original copy of a treaty or an agreement to MOFA for filing purpose pursuant to Article 15 of the Act, competent authorities other than MOFA shall submit the Record Form for Treaties or Agreements Filing, editable electronic files of the treaty or agreement, and a copy of its Chinese translation.]]></條文內容>
  </條文>
  <條文>
    <條號>Article 11</條號>
    <條文內容><![CDATA[The Enforcement Rules shall become effective on the date of promulgation.]]></條文內容>
  </條文>
</法規內容>
</法規>

redpajama_set_name: RedPajamaGithub


text:
package edu.arizona.sista.utils

import org.scalatest._

/**
 *
 * User: mihais
 * Date: 4/3/13
 */
class TestArgsToProperties extends FlatSpec with Matchers {
  val props = StringUtils.argsToProperties(List("-props", "src/test/resources/edu/arizona/sista/utils/test.properties").toArray)

  "properties" should "contain p1 with value /some/path" in {
    val p1 = props.getProperty("p1")
    p1 should be ("/some/path")
  }

  it should "contain p2 with the value /some/path/subdir/123" in {
    val p2 = props.getProperty("p2")
    p2 should be ("/some/path/subdir/123")
  }

  it should "contain p4 set to a valid shell" in {
    val p4 = props.getProperty("p4")
    p4 should fullyMatch regex """shell is /bin/bash|shell is /bin/sh|shell is /bin/csh|shell is /bin/tcsh|shell is /bin/zsh"""
  }
}


text:
class Account < ActiveRecord::Base
  belongs_to :owner, :class_name => 'User', :foreign_key => 'owner_id'

  has_many :users

  def self.owner(reload=false)
    @@cached_owner = nil if reload
    @@cached_owner ||= Account.first
  end

  attr_accessible :site_name, :host_name

  Tabs = ['overview', 'pages', 'reminders', 'journal']

  Tabs.each do |tab|
    define_method("#{tab}_hidden?") do
      value = self.get_setting("#{tab}_hidden")
      return value == '1' || value == 1 || value == true || value == 'true'
    end

    define_method("#{tab}_hidden") do
      return self.get_setting("#{tab}_hidden")
    end

    define_method("#{tab}_hidden=") do |value|
      self.set_setting("#{tab}_hidden", value)
    end

    attr_accessible "#{tab}_hidden", "#{tab}_hidden?"
  end

  Colours = ['header', 'tab_background', 'tab_text', 'tab_background_hover', 'tab_text_hover', 'page_header', 'page_header_text']

  Colours.each do |colour|
    define_method("#{colour}_colour") do
      value = self.get_setting("#{colour}_colour")
      return value if value != nil && value.length > 0
      return self.send("default_#{colour}_colour")
    end

    define_method("#{colour}_colour=") do |value|
      self.set_setting("#{colour}_colour", value)
    end

    define_method("default_#{colour}_colour") do
      case colour
      when "header"
        "#007700"
      when "tab_background"
        "#006600"
      when "tab_text"
        "#ffffff"
      when "page_header"
        "#e0eedf"
      when "page_header_text"
        "#333333"
      else
        ""
      end
    end

    attr_accessible "#{colour}_colour"
  end

  # Settings Serialization
  def get_setting(key)
    (self.settings_hash)[key]
  end

  def set_setting(key, value)
    hash = self.settings_hash
    hash[key] = value
    self.settings = YAML.dump(hash)
  end

  def settings_hash
    if self.settings == nil || self.settings.length <= 0
      return Hash.new
    else
      return YAML.load(self.settings)
    end
  end
end