text:
#!/usr/bin/env python
"""Simple parsers for configuration files."""
import collections
import re

import logging

from grr.lib import lexer
from grr.lib import parsers
from grr.lib import utils
from grr.lib.rdfvalues import anomaly as rdf_anomaly
from grr.lib.rdfvalues import client as rdf_client
from grr.lib.rdfvalues import config_file as rdf_config_file
from grr.lib.rdfvalues import protodict as rdf_protodict
from grr.lib.rdfvalues import standard as rdf_standard

def AsIter(arg):
  """Encapsulates an argument in a tuple, if it's not already iterable."""
  if isinstance(arg, basestring):
    rslt = [arg]
  elif isinstance(arg, collections.Iterable):
    rslt = arg
  elif not arg:
    rslt = []
  else:
    rslt = [arg]
  return tuple(rslt)

# Grr lexer implementation of ssv parser. Considered using
# https://github.com/Eugeny/reconfigure/blob/master/reconfigure/parsers/ssv.py
# but it doesn't seem to actually forward lookup.
class FieldParser(lexer.Lexer):
  r"""A generalized field based parser that splits entries into fields.

  Entries refer to distinct records within the text content, for example each
  line of /etc/passwd or a ssh configuration attribute.
  Fields are elements that make up the entry, for example the individual
  parameters in /etc/passwd.

  The parser supports:
  - Flexible field based separators (e.g. spaces, commas, colons).
  - Identification and removal of line comments. Inline comments (e.g. /*...*/)
    are not supported.
  - Line continuation detection.
  - Multiline quotes.

  The parser uses the following attributes as defaults:
  - comments: #
  - cont: \ (followed by any amount of whitespace)
  - ml_quote: False (by default, quotes must close before newlines).
  - quot: Both " and ' characters.
  - sep: Whitespace
  - term: Newlines.

  To override default values, pass in appropriate keywords with a python
  compatible regex string.
  """

  def __init__(self,
               comments=r"#",
               cont=r"\\\s*\n",
               ml_quote=False,
               quot=(r"\"", r"'"),
               sep=r"[ \t\f\v]+",
               term=r"[\r\n]",
               verbose=0):
    r"""A generalized field-based parser. Handles whitespace, csv etc.

    Args:
      comments: Line comment patterns (e.g. "#").
      cont: Continuation patterns (e.g. "\\").
      ml_quote: Boolean flag to allow quoted strings to span lines.
      quot: Quotation patterns (e.g. "\\"" or "'").
      sep: Field separator patterns (e.g. "[\\s,]").
      term: Entry termination patterns (e.g. "\\n").
      verbose: Enable verbose mode for the lexer. Useful for debugging.
    """
    super(FieldParser, self).__init__()
    self.entries = []
    self.fields = []
    self.field = ""
    self.comments = AsIter(comments)
    self.cont = AsIter(cont)
    self.ml_quote = AsIter(ml_quote)
    self.quot = AsIter(quot)
    self.sep = AsIter(sep)
    self.term = AsIter(term)
    self.verbose = verbose
    self._GenStates()

  def Reset(self):
    super(FieldParser, self).Reset()
    self.entries = []
    self.fields = []
    self.field = ""

  def _GenStates(self):
    """Generate the lexer states."""
    self.GenCommentState()
    self.GenFwdState()
    self.GenQuotedState()
    self.GenCatchallState()

  def _AddToken(self, state_regex, regex, actions, next_state):
    self._tokens.append(lexer.Token(state_regex, regex, actions, next_state))

  def GenCommentState(self):
    if self.comments:
      self._AddToken("COMMENT", r"\n", "PushBack,PopState", None)
      self._AddToken("COMMENT", ".", None, None)

  def GenFwdState(self):
    """Generates forwarding state rules.

    The lexer will fast forward until there is string content. The
    string content will be returned to the string processor.
    """
    for c in self.cont:
      self._AddToken("FWD", c, None, None)
    for s in self.sep:
      self._AddToken("FWD", s, None, None)
    self._AddToken("FWD", ".", "PushBack,PopState", None)

  def GenQuotedState(self):
    """Generate string matching state rules."""
    for i, q in enumerate(self.quot):
      label = "%s_STRING" % i
      escaped = q.encode("string_escape")
      self._AddToken(label, escaped, "PopState", None)
      self._AddToken(label, q, "PopState", None)
      if self.ml_quote:
        self._AddToken(label, r"\n", None, None)
      else:
        self._AddToken(label, r"\n", "BadLine", None)
      self._AddToken(label, ".", "AddToField", None)

  def GenCatchallState(self):
    """Generate string matching state rules.

    This sets up initial state handlers that cover both the 'INITIAL' state
    and the intermediate content between fields.

    The lexer acts on items with precedence:
      - continuation characters: use the fast forward state rules.
      - field separators: finalize processing the field.
      - quotation characters: use the quotation state rules.
    """
    for c in self.comments:
      self._AddToken(".", c, "PushState,EndField", "COMMENT")
    for c in self.cont:
      self._AddToken(".", c, "PushState", "FWD")
    for t in self.term:
      self._AddToken(".", t, "EndEntry", None)
    for s in self.sep:
      self._AddToken(".", s, "EndField", None)
    for i, q in enumerate(self.quot):
      self._AddToken(".", q, "PushState", "%s_STRING" % i)
    self._AddToken(".", ".", "AddToField", None)

  def EndEntry(self, **_):
    self.EndField()
    if self.fields:
      # Copy the fields into the processed entries.
      self.entries.append(self.fields[:])
    self.fields = []

  def AddToField(self, string="", **_):
    if string:
      self.field += string

  def EndField(self, **_):
    if self.field:
      self.fields.append(self.field[:])
      self.field = ""

  def BadLine(self, **_):
    logging.debug("Skipped bad line in file at %s", self.processed)
    self.field = ""

  def ParseEntries(self, data):
    # Flush any old results.
    self.Reset()
    self.Feed(utils.SmartStr(data))
    self.Close()
    # In case there isn't a terminating field at the end of the feed, e.g. \n
    self.EndEntry()
    return self.entries

class KeyValueParser(FieldParser):
  """A generalized KeyValue parser that splits entries into key/value pairs.

  Capabilities and parameters are identical to FieldParser, with one difference.
  The parser also accepts the parameter "kv_sep"
  Patterns specified in kv_sep are used to demarcate key/value processing.

  kv_sep defaults to "="
  """

  def __init__(self,
               comments=r"#",
               cont=r"\\\s*\n",
               kv_sep="=",
               ml_quote=False,
               quot=(r"\"", r"'"),
               sep=r"[ \t\f\v]+",
               term=r"[\r\n]",
               verbose=0):
    """A generalized key-value parser. Handles whitespace, csv etc.

    Args:
      comments: Line comment patterns (e.g. "#").
      cont: Continuation patterns (e.g. "\\").
      kv_sep: Key/Value separators (e.g. "=" or ":").
      ml_quote: Boolean flag to allow quoted strings to span lines.
      quot: Quotation patterns (e.g. "\\"" or "'").
      sep: Field separator patterns (e.g. "[\\s,]").
      term: Entry termination patterns (e.g. "\\n").
      verbose: Enable verbose mode for the lexer. Useful for debugging.
    """
    self.kv_sep = AsIter(kv_sep)
    super(KeyValueParser, self).__init__(
        comments=comments,
        cont=cont,
        ml_quote=ml_quote,
        quot=quot,
        sep=sep,
        term=term,
        verbose=verbose)
    self.key_field = ""

  def _GenStates(self):
    self.GenCommentState()
    self.GenFwdState()
    self.GenQuotedState()
    self.GenMatchFirstState()
    self.GenInitialState()
    self.GenKeyState()
    self.GenValueState()
    self.GenCatchallState()

  def GenMatchFirstState(self):
    for i, q in enumerate(self.quot):
      self._AddToken(".", q, "PushState", "%s_STRING" % i)
    for c in self.cont:
      self._AddToken(".", c, "PushState", "FWD")

  def GenInitialState(self):
    for c in self.comments:
      self._AddToken("INITIAL", c, "PushState,EndField", "COMMENT")
    for t in self.term:
      self._AddToken("INITIAL", t, "EndField,EndEntry", None)
    for c in self.sep:
      self._AddToken("INITIAL", c, "PushState", "FWD")
    for k in self.kv_sep:
      self._AddToken("INITIAL", k, "BadLine", None)
    self._AddToken("INITIAL", ".", "PushState,PushBack", "KEY")

  def GenKeyState(self):
    for c in self.comments:
      self._AddToken("KEY", c, "EndKeyField,EndEntry,PopState,PushBack",
                     "COMMENT")
    for t in self.term:
      self._AddToken("KEY", t, "EndKeyField,EndEntry,PopState", None)
    for k in self.kv_sep:
      self._AddToken("KEY", k, "EndKeyField", "VALUE")

  def GenValueState(self):
    for c in self.comments:
      self._AddToken("VALUE", c, "EndField,EndEntry,PopState,PushBack",
                     "COMMENT")
    for t in self.term:
      self._AddToken("VALUE", t, "EndField,EndEntry,PopState", None)
    for s in self.sep:
      self._AddToken("VALUE", s, "EndField", None)

  def GenCatchallState(self):
    self._AddToken(".", ".", "AddToField", None)

  def EndKeyField(self, **_):
    self.key_field = self.field
    self.field = ""

  def EndEntry(self, **_):
    # Finalize processing for non-terminated entries. Key first, then fields.
    if self.field and not self.key_field:
      self.EndKeyField()
    else:
      self.EndField()
    # Set up the entry.
    key_field = self.key_field.strip()
    if key_field:
      self.entries.append({key_field: self.fields})
    self.key_field = ""
    self.fields = []

  def ParseToOrderedDict(self, data):
    result = collections.OrderedDict()
    for field in self.ParseEntries(data):
      result.update(field)
    return result

class NfsExportsParser(parsers.FileParser, FieldParser):
  """Parser for NFS exports."""

  output_types = ["NfsExport"]
  supported_artifacts = ["NfsExportsFile"]

  def Parse(self, unused_stat, file_obj, unused_knowledge_base):
    for entry in self.ParseEntries(file_obj.read()):
      if not entry:
        continue
      result = rdf_config_file.NfsExport()
      result.share = entry[0]
      for field in entry[1:]:
        if field.startswith(("-", "(")):
          result.defaults = field.strip("-()").split(",")
        else:
          client = rdf_config_file.NfsClient()
          cfg = field.split("(", 1)
          host = cfg[0]
          if len(cfg) > 1:
            options = cfg[1]
          else:
            options = None
          client.host = host
          if options:
            client.options = options.strip("()").split(",")
          result.clients.append(client)
      yield result

class SshdParserBase(object):
  """The base class for the ssh config parsers."""

  output_types = ["SshdConfig"]
  # Specify the values that are boolean or integer. Anything else is a string.
  _integers = ["clientalivecountmax",
               "magicudsport",
               "maxauthtries",
               "maxsessions",
               "port",
               "protocol",
               "serverkeybits",
               "x11displayoffset"]  # pyformat: disable
  _booleans = ["allowagentforwarding",
               "challengeresponseauthentication",
               "dsaauthentication"
               "gssapiauthentication",
               "gssapicleanupcredentials",
               "gssapikeyexchange",
               "gssapistorecredentialsonrekey",
               "gssapistrictacceptorcheck",
               "hostbasedauthentication",
               "ignorerhosts",
               "ignoreuserknownhosts",
               "kbdinteractiveauthentication",
               "kerberosauthentication",
               "passwordauthentication",
               "permitemptypasswords",
               "permittunnel",
               "permituserenvironment",
               "pubkeyauthentication",
               "rhostsrsaauthentication",
               "rsaauthentication",
               "strictmodes",
               "uselogin",
               "usepam",
               "x11forwarding",
               "x11uselocalhost"]  # pyformat: disable
  # Valid ways that parameters can repeat
  _repeated = {
      "acceptenv": r"[\n\s]+",
      "allowgroups": r"[\s]+",
      "allowusers": r"[\s]+",
      "authenticationmethods": r"[\s]+",
      "authorizedkeysfile": r"[\s]+",
      "ciphers": r"[,]+",
      "denygroups": r"[\s]+",
      "denyusers": r"[\s]+",
      "forcecommand": r"[\n]+",
      "hostkey": r"[\n]+",
      "kexalgorithms": r"[,]+",
      "listenaddress": r"[\n]+",
      "macs": r"[,]+",
      "permitopen": r"[\s]+",
      "port": r"[,\n]+",
      "protocol": r"[,]+",
      "pubkeyacceptedkeytypes": r"[,]+",
      "subsystem": r"[\n]+"
  }
  _true = ["yes", "true", "1"]
  _aliases = {"dsaauthentication": "pubkeyauthentication"}
  _match_keywords = [
      "acceptenv", "allowagentforwarding", "allowgroups", "allowtcpforwarding",
      "allowusers", "authenticationmethods", "authorizedkeyscommand",
      "authorizedkeyscommanduser", "authorizedkeysfile",
      "authorizedprincipalsfile", "banner", "chrootdirectory", "denygroups",
      "denyusers", "forcecommand", "gatewayports", "gssapiauthentication",
      "hostbasedauthentication", "hostbasedusesnamefrompacketonly",
      "kbdinteractiveauthentication", "kerberosauthentication", "magicudspath",
      "magicudsport", "maxauthtries", "maxsessions", "passwordauthentication",
      "permitemptypasswords", "permitopen", "permitrootlogin",
      "permittemphomedir", "permittty", "permittunnel",
      "pubkeyacceptedkeytypes", "pubkeyauthentication", "rekeylimit",
      "rhostsrsaauthentication", "rsaauthentication", "temphomedirpath",
      "x11displayoffset", "x11forwarding", "x11uselocalhost"
  ]

  def __init__(self):
    super(SshdParserBase, self).__init__()
    self.Flush()

  def Flush(self):
    self.config = {}
    self.matches = []
    self.section = self.config
    self.processor = self._ParseEntry

  def ParseLine(self, line):
    """Extracts keyword/value settings from the sshd config.

    The keyword is always the first string item.
    Values are the remainder of the string. In cases where an sshd config
    allows multiple values, these are split according to whatever separator(s)
    sshd_config permits for that value.

    Keywords and values are normalized. Keywords are converted to lowercase.
    Values are converted into integers, booleans or strings. Strings are always
    lowercased.

    Args:
      line: A line of the configuration file.
    """
    kv = line.split(None, 1)
    keyword = kv[0].lower()
    # Safely set the argument string if it wasn't found.
    values = kv[1:] or [""]
    # Then split any parameters that are actually repeated items.
    separators = self._repeated.get(keyword)
    if separators:
      repeated = []
      for v in values:
        repeated.extend(re.split(separators, v))
      # Remove empty matches.
      values = [v for v in repeated if v]

    # Now convert the values to the right types.
    if keyword in self._integers:
      values = [int(v) for v in values]
    elif keyword in self._booleans:
      values = [v.lower() in self._true for v in values]
    else:
      values = [v.lower() for v in values]
    # Only repeated arguments should be treated as a list.
    if keyword not in self._repeated:
      values = values[0]
    # Switch sections for new match blocks.
    if keyword == "match":
      self._NewMatchSection(values)
    # If it's an alias, resolve it.
    if keyword in self._aliases:
      keyword = self._aliases[keyword]
    # Add the keyword/values to the section.
    self.processor(keyword, values)

  def _ParseEntry(self, key, val):
    """Adds an entry for a configuration setting.

    Args:
      key: The name of the setting.
      val: The value of the setting.
    """
    if key in self._repeated:
      setting = self.section.setdefault(key, [])
      setting.extend(val)
    else:
      self.section.setdefault(key, val)

  def _ParseMatchGrp(self, key, val):
    """Adds valid match group parameters to the configuration."""
    if key in self._match_keywords:
      self._ParseEntry(key, val)

  def _NewMatchSection(self, val):
    """Create a new configuration section for each match clause.

    Each match clause is added to the main config, and the criterion that will
    trigger the match is recorded, as is the configuration.

    Args:
      val: The value following the 'match' keyword.
    """
    section = {"criterion": val, "config": {}}
    self.matches.append(section)
    # Now add configuration items to config section of the match block.
    self.section = section["config"]
    # Switch to a match-specific processor on a new match_block.
    self.processor = self._ParseMatchGrp

  def GenerateResults(self):
    matches = []
    for match in self.matches:
      criterion, config = match["criterion"], match["config"]
      block = rdf_config_file.SshdMatchBlock(criterion=criterion, config=config)
      matches.append(block)
    yield rdf_config_file.SshdConfig(config=self.config, matches=matches)

class SshdConfigParser(SshdParserBase, parsers.FileParser):
  """A parser for sshd_config files."""

  supported_artifacts = ["SshdConfigFile"]

  def Parse(self, stat, file_object, knowledge_base):
    """Parse the sshd configuration.

    Process each of the lines in the configuration file.

    Assembes an sshd_config file into a dictionary with the configuration
    keyword as the key, and the configuration settings as value(s).

    Args:
      stat: unused
      file_object: An open configuration file object.
      knowledge_base: unused

    Yields:
      The configuration as an rdfvalue.
    """
    _, _ = stat, knowledge_base
    # Clean out any residual state.
    self.Flush()
    lines = [l.strip() for l in file_object.read(100000).splitlines()]
    for line in lines:
      # Remove comments (will break if it includes a quoted/escaped #)
      line = line.split("#")[0].strip()
      if line:
        self.ParseLine(line)
    for result in self.GenerateResults():
      yield result

class SshdConfigCmdParser(SshdConfigParser, parsers.CommandParser):
  """A command parser for sshd -T output."""

  supported_artifacts = ["SshdConfigCmd"]

  def Parse(self, cmd, args, stdout, stderr, return_val, time_taken,
            knowledge_base):
    # Clean out any residual state.
    self.Flush()
    lines = [l.strip() for l in stdout.splitlines()]
    for line in lines:
      if line:
        self.ParseLine(line)
    for result in self.GenerateResults():
      yield result

class MtabParser(parsers.FileParser, FieldParser):
  """Parser for mounted filesystem data acquired from /proc/mounts."""
  output_types = ["Filesystem"]
  supported_artifacts = ["LinuxProcMounts", "LinuxFstab"]

  def Parse(self, unused_stat, file_obj, unused_knowledge_base):
    for entry in self.ParseEntries(file_obj.read()):
      if not entry:
        continue
      result = rdf_client.Filesystem()
      result.device = entry[0].decode("string_escape")
      result.mount_point = entry[1].decode("string_escape")
      result.type = entry[2].decode("string_escape")
      options = KeyValueParser(term=",").ParseToOrderedDict(entry[3])
      # Keys without values get assigned [] by default. Because these keys are
      # actually true, if declared, change any [] values to True.
      for k, v in options.iteritems():
        options[k] = v or [True]
      result.options = rdf_protodict.AttributedDict(**options)
      yield result

class MountCmdParser(parsers.CommandParser, FieldParser):
  """Parser for mounted filesystem data acquired from the mount command."""
  output_types = ["Filesystem"]
  supported_artifacts = ["LinuxMountCmd"]

  mount_re = re.compile(r"(.*) on (.*) type (.*) \((.*)\)")

  def Parse(self, cmd, args, stdout, stderr, return_val, time_taken,
            knowledge_base):
    """Parse the mount command output."""
    _ = stderr, time_taken, args, knowledge_base  # Unused.
    self.CheckReturn(cmd, return_val)
    result = rdf_protodict.AttributedDict()
    for entry in self.ParseEntries(stdout):
      line_str = " ".join(entry)
      mount_rslt = self.mount_re.match(line_str)
      if mount_rslt:
        device, mount_point, fs_type, option_str = mount_rslt.groups()
        result = rdf_client.Filesystem()
        result.device = device
        result.mount_point = mount_point
        result.type = fs_type
        # Parse these options as a dict as some items may be key/values.
        # KeyValue parser uses OrderedDict as the native parser method. Use it.
        options = KeyValueParser(term=",").ParseToOrderedDict(option_str)
        # Keys without values get assigned [] by default. Because these keys are
        # actually true, if declared, change any [] values to True.
        for k, v in options.iteritems():
          options[k] = v or [True]
        result.options = rdf_protodict.AttributedDict(**options)
        yield result

class RsyslogParser(parsers.FileParser, FieldParser):
  """Parser for syslog configurations."""
  output_types = ["AttributedDict"]
  supported_artifacts = ["LinuxRsyslogConfigs"]
  process_together = True

  log_rule_re = re.compile(r"([\w,\*]+)\.([\w,!=\*]+)")
  destinations = collections.OrderedDict([
      ("TCP", re.compile(r"(?:@@)([^;]*)")),
      ("UDP", re.compile(r"(?:@)([^;]*)")),
      ("PIPE", re.compile(r"(?:\|)([^;]*)")),
      ("NONE", re.compile(r"(?:~)([^;]*)")),
      ("SCRIPT", re.compile(r"(?:\^)([^;]*)")),
      ("MODULE", re.compile(r"(?::om\w:)([^;]*)")),
      ("FILE", re.compile(r"-?(/[^;]*)")), ("WALL", re.compile(r"(\*)"))
  ])  # pyformat: disable

  def _ParseAction(self, action):
    """Extract log configuration data from rsyslog actions.

    Actions have the format:
      <facility>/<severity> <type_def><destination>;<template>
      e.g. *.* @@loghost.example.com.:514;RSYSLOG_ForwardFormat

    Actions are selected by a type definition. These include:
      "@@": TCP syslog
      "@": UDP syslog
      "|": Named pipe
      "~": Drop to /dev/null
      "^": Shell script
      ":om<string>:": An output module
      Or a file path.

    Args:
      action: The action string from rsyslog.

    Returns:
      a rdfvalue.LogTarget message.
    """
    rslt = rdf_config_file.LogTarget()
    for dst_str, dst_re in self.destinations.iteritems():
      dst = dst_re.match(action)
      if dst:
        rslt.transport = dst_str
        rslt.destination = dst.group(1)
        break
    return rslt

  def ParseMultiple(self, unused_stats, file_objs, unused_knowledge_base):
    # TODO(user): review quoting and line continuation.
    result = rdf_config_file.LogConfig()
    for file_obj in file_objs:
      for entry in self.ParseEntries(file_obj.read()):
        directive = entry[0]
        log_rule = self.log_rule_re.match(directive)
        if log_rule and entry[1:]:
          target = self._ParseAction(entry[1])
          target.facility, target.priority = log_rule.groups()
          result.targets.append(target)
    return [result]

class PackageSourceParser(parsers.FileParser):
  """Common code for APT and YUM source list parsing."""
  output_types = ["AttributedDict"]

  # Prevents this from automatically registering.
  __abstract = True  # pylint: disable=g-bad-name

  def Parse(self, stat, file_obj, unused_knowledge_base):
    uris_to_parse = self.FindPotentialURIs(file_obj)
    uris = []

    for url_to_parse in uris_to_parse:
      url = rdf_standard.URI()
      url.ParseFromString(url_to_parse)

      # if no transport then url_to_parse wasn't actually a valid URL
      # either host or path also have to exist for this to be a valid URL
      if url.transport and (url.host or url.path):
        uris.append(url)

    filename = stat.pathspec.path
    cfg = {"filename": filename, "uris": uris}
    yield rdf_protodict.AttributedDict(**cfg)

  def FindPotentialURIs(self, file_obj):
    """Stub Method to be overriden by APT and Yum source parsers."""
    raise NotImplementedError("Please implement FindPotentialURIs.")

  def ParseURIFromKeyValues(self, data, separator, uri_key):
    """Parse key/value formatted source listing and return potential URLs.

    The fundamental shape of this format is as follows:
    key: value   # here : = separator
    key : value
    URI: [URL]   # here URI = uri_key
      [URL]      # this is where it becomes trickey because [URL]
      [URL]      # can contain 'separator' specially if separator is :
    key: value

    The key uri_key is of interest to us and since the next line
    in the config could contain another [URL], we need to keep track of context
    when we hit uri_key  to be able to check if the next line(s)
    have more [URL].

    Args:
      data: unprocessed lines from a file
      separator: how the key/value pairs are seperated
      uri_key: starting name of the key containing URI.

    Returns:
      A list of potential URLs found in data
    """
    kv_entries = KeyValueParser(kv_sep=separator).ParseEntries(data)
    spaced_entries = FieldParser().ParseEntries(data)

    uris = []
    check_uri_on_next_line = False
    for kv_entry, sp_entry in zip(kv_entries, spaced_entries):
      for k, v in kv_entry.iteritems():
        # This line could be a URL if a) from  key:value, value is empty OR
        # b) if separator is : and first character of v starts with /.
        if (check_uri_on_next_line and
            (not v or (separator == ":" and v[0].startswith("/")))):
          uris.append(sp_entry[0])
        else:
          check_uri_on_next_line = False
          if k.lower().startswith(uri_key) and v:
            check_uri_on_next_line = True
            uris.append(v[0])  # v is a list

    return uris

class APTPackageSourceParser(PackageSourceParser):
  """Parser for APT source lists to extract URIs only."""
  supported_artifacts = ["APTSources"]

  def FindPotentialURIs(self, file_obj):
    """Given a file, this will return all potenial APT source URIs."""
    rfc822_format = ""  # will contain all lines not in legacy format
    uris_to_parse = []

    for line in file_obj.read().splitlines(True):
      # check if legacy style line - if it is then extract URL
      m = re.search(r"^\s*deb(?:-\S+)?(?:\s+\[[^\]]*\])*\s+(\S+)(?:\s|$)", line)
      if m:
        uris_to_parse.append(m.group(1))
      else:
        rfc822_format += line

    uris_to_parse.extend(self.ParseURIFromKeyValues(rfc822_format, ":", "uri"))
    return uris_to_parse

class YumPackageSourceParser(PackageSourceParser):
  """Parser for Yum source lists to extract URIs only."""
  supported_artifacts = ["YumSources"]

  def FindPotentialURIs(self, file_obj):
    """Given a file, this will return all potenial Yum source URIs."""
    return self.ParseURIFromKeyValues(file_obj.read(), "=", "baseurl")

class CronAtAllowDenyParser(parsers.FileParser):
  """Parser for /etc/cron.allow /etc/cron.deny /etc/at.allow & /etc/at.deny."""
  output_types = ["AttributedDict"]
  supported_artifacts = ["CronAtAllowDenyFiles"]

  def Parse(self, stat, file_obj, unused_knowledge_base):
    lines = set([l.strip() for l in file_obj.read(100000).splitlines()])

    users = []
    bad_lines = []
    for line in lines:
      # behaviour of At/Cron is undefined for lines with whitespace separated
      # fields/usernames
      if " " in line:
        bad_lines.append(line)
      elif line:  # drop empty lines
        users.append(line)

    filename = stat.pathspec.path
    cfg = {"filename": filename, "users": users}
    yield rdf_protodict.AttributedDict(**cfg)

    if bad_lines:
      yield rdf_anomaly.Anomaly(
          type="PARSER_ANOMALY",
          symptom="Dodgy entries in %s." % (filename),
          reference_pathspec=stat.pathspec,
          finding=bad_lines)

class NtpdParser(parsers.FileParser, FieldParser):
  """Parser for ntpd.conf file."""
  output_types = ["NtpConfig"]
  supported_artifacts = ["NtpConfFile"]
  process_together = True

  # The syntax is based on:
  #   https://www.freebsd.org/cgi/man.cgi?query=ntp.conf&sektion=5
  # keywords with integer args.
  _integers = set(["ttl", "hop"])
  # keywords with floating point args.
  _floats = set(["broadcastdelay", "calldelay"])
  # keywords that have repeating args.
  _repeated = set(["ttl", "hop"])
  # keywords that set an option state, but can be "repeated" as well.
  _boolean = set(["enable", "disable"])
  # keywords that are keyed to their first argument, an address.
  _address_based = set([
      "trap", "fudge", "server", "restrict", "peer", "broadcast",
      "manycastclient"
  ])
  # keywords that append/augment the config.
  _accumulators = set(["includefile", "setvar"])
  # keywords that can appear multiple times, accumulating data each time.
  _duplicates = _address_based | _boolean | _accumulators
  # All the expected keywords.
  _match_keywords = _integers | _floats | _repeated | _duplicates | set([
      "autokey", "revoke", "multicastclient", "driftfile", "broadcastclient",
      "manycastserver", "includefile", "interface", "disable", "includefile",
      "discard", "logconfig", "logfile", "tos", "tinker", "keys", "keysdir",
      "requestkey", "trustedkey", "crypto", "control", "statsdir", "filegen"
  ])

  _defaults = {
      "auth": True,
      "bclient": False,
      "calibrate": False,
      "kernel": False,
      "monitor": True,
      "ntp": True,
      "pps": False,
      "stats": False
  }

  def ParseLine(self, entries):
    """Extracts keyword/value settings from the ntpd config.

    The keyword is always the first entry item.
    Values are the remainder of the entries. In cases where an ntpd config
    allows multiple values, these are split according to whitespace or
    duplicate entries.

    Keywords and values are normalized. Keywords are converted to lowercase.
    Values are converted into integers, floats or strings. Strings are always
    lowercased.

    Args:
      entries: A list of items making up a single line of a ntp.conf file.
    """
    # If no entries were found, short circuit.
    if not entries:
      return
    keyword = entries[0].lower()
    # Set the argument string if it wasn't found.
    values = entries[1:] or [""]

    # Convert any types we need too.
    if keyword in self._integers:
      values = [int(v) for v in values]
    if keyword in self._floats:
      values = [float(v) for v in values]

    if keyword not in self._repeated | self._duplicates:
      # We have a plain and simple single key/value config line.
      if isinstance(values[0], basestring):
        self.config[keyword] = " ".join(values)
      else:
        self.config[keyword] = values

    elif keyword in self._repeated:
      # The keyword can have multiple single-word options, so add them as a list
      # and overwrite previous settings.
      self.config[keyword] = values

    elif keyword in self._duplicates:
      if keyword in self._address_based:
        # If we have an address keyed keyword, join the keyword and address
        # together to make the complete key for this data.
        address = values[0].lower()
        values = values[1:] or [""]
        # Add/overwrite the address in this 'keyed' keywords dictionary.
        existing_keyword_config = self.keyed.setdefault(keyword, [])
        # Create a dict which stores the server name and the options.
        # Flatten the remaining options into a single string.
        existing_keyword_config.append({
            "address": address,
            "options": " ".join(values)
        })

      # Are we toggling an option?
      elif keyword in self._boolean:
        for option in values:
          if keyword == "enable":
            self.config[option] = True
          else:
            # As there are only two items in this set, we can assume disable.
            self.config[option] = False

      else:
        # We have a non-keyed & non-boolean keyword, so add to the collected
        # data so far. Order matters technically.
        prev_settings = self.config.setdefault(keyword, [])
        prev_settings.append(" ".join(values))

  def Parse(self, stat, file_object, knowledge_base):
    """Parse a ntp config into rdf."""
    _, _ = stat, knowledge_base
    # Clean out any residual state.
    self.config = self._defaults.copy()
    self.keyed = {}
    # ntp.conf has no line continuation. Override the default 'cont' values
    # then parse up the lines.
    self.cont = ""
    for line in self.ParseEntries(file_object.read(100000)):
      self.ParseLine(line)
    yield rdf_config_file.NtpConfig(
        config=self.config,
        server=self.keyed.get("server"),
        restrict=self.keyed.get("restrict"),
        fudge=self.keyed.get("fudge"),
        trap=self.keyed.get("trap"),
        peer=self.keyed.get("peer"),
        broadcast=self.keyed.get("broadcast"),
        manycastclient=self.keyed.get("manycastclient"))

  def ParseMultiple(self, stats, file_objects, knowledge_base):
    for s, f in zip(stats, file_objects):
      for rslt in self.Parse(s, f, knowledge_base):
        yield rslt

class SudoersParser(parsers.FileParser, FieldParser):
  """Parser for privileged configuration files such as sudoers and pam.d/su."""
  output_types = ["SudoersConfig"]
  supported_artifacts = ["UnixSudoersConfiguration"]

  # Regex to remove comments from the file. The first group in the OR condition
  # handles comments that cover a full line, while also ignoring #include(dir).
  # The second group in the OR condition handles comments that begin partways
  # through a line, without matching UIDs or GIDs which are specified with # in
  # the format.
  # TODO(user): this regex fails to match '#32 users', but handles quite a
  # lot else.
  # TODO(user): this should be rewritten as a proper lexer
  COMMENTS_RE = re.compile(r"(#(?!include(?:dir)?\s+)\D+?$)", re.MULTILINE)

  ALIAS_TYPES = {
      "User_Alias": rdf_config_file.SudoersAlias.Type.USER,
      "Runas_Alias": rdf_config_file.SudoersAlias.Type.RUNAS,
      "Host_Alias": rdf_config_file.SudoersAlias.Type.HOST,
      "Cmnd_Alias": rdf_config_file.SudoersAlias.Type.CMD
  }
  ALIAS_FIELDS = {
      "User_Alias": "users",
      "Runas_Alias": "runas",
      "Host_Alias": "hosts",
      "Cmnd_Alias": "cmds"
  }
  DEFAULTS_KEY = "Defaults"
  INCLUDE_KEYS = ["#include", "#includedir"]

  def __init__(self, *args, **kwargs):
    kwargs["comments"] = []
    super(SudoersParser, self).__init__(*args, **kwargs)

  def _ExtractList(self, fields, ignores=(",",), terminators=()):
    """Extract a list from the given fields."""
    extracted = []
    i = 0
    for i, field in enumerate(fields):
      # Space-separated comma; ignore, but this is not a finished list.
      # Similar for any other specified ignores (eg, equals sign).
      if field in ignores:
        continue

      # However, some fields are specifically meant to terminate iteration.
      if field in terminators:
        break

      extracted.append(field.strip("".join(ignores)))
      # Check for continuation; this will either be a trailing comma or the
      # next field after this one being a comma. The lookahead here is a bit
      # nasty.
      if not (field.endswith(",") or set(
          fields[i + 1:i + 2]).intersection(ignores)):
        break

    return extracted, fields[i + 1:]

  def _ParseSudoersEntry(self, entry, sudoers_config):
    """Parse an entry and add it to the given SudoersConfig rdfvalue."""

    key = entry[0]
    if key in SudoersParser.ALIAS_TYPES.keys():
      # Alias.
      alias_entry = rdf_config_file.SudoersAlias(
          type=SudoersParser.ALIAS_TYPES.get(key), name=entry[1])

      # Members of this alias, comma-separated.
      members, _ = self._ExtractList(entry[2:], ignores=(",", "="))
      field = SudoersParser.ALIAS_FIELDS.get(key)
      getattr(alias_entry, field).Extend(members)

      sudoers_config.aliases.append(alias_entry)
    elif key.startswith(SudoersParser.DEFAULTS_KEY):
      # Default.
      # Identify scope if one exists (Defaults<scope> ...)
      scope = None
      if len(key) > len(SudoersParser.DEFAULTS_KEY):
        scope = key[len(SudoersParser.DEFAULTS_KEY) + 1:]

      # There can be multiple defaults on a line, for the one scope.
      entry = entry[1:]
      defaults, _ = self._ExtractList(entry)
      for default in defaults:
        default_entry = rdf_config_file.SudoersDefault(scope=scope)

        # Extract key name and value(s).
        default_name = default
        value = []
        if "=" in default_name:
          default_name, remainder = default_name.split("=", 1)
          value = [remainder]
        default_entry.name = default_name
        if entry:
          default_entry.value = " ".join(value)

        sudoers_config.defaults.append(default_entry)
    elif key in SudoersParser.INCLUDE_KEYS:
      # TODO(user): make #includedir more obvious in the RDFValue somewhere
      target = " ".join(entry[1:])
      sudoers_config.includes.append(target)
    else:
      users, entry = self._ExtractList(entry)
      hosts, entry = self._ExtractList(entry, terminators=("=",))

      # Remove = from <user> <host> = <specs>
      if entry[0] == "=":
        entry = entry[1:]

      # Command specification.
      sudoers_entry = rdf_config_file.SudoersEntry(
          users=users, hosts=hosts, cmdspec=entry)

      sudoers_config.entries.append(sudoers_entry)

  def _Preprocess(self, data):
    """Preprocess the given data, ready for parsing."""
    # Add whitespace to line continuations.
    data = data.replace(":\\", ": \\")

    # Strip comments manually because sudoers has multiple meanings for '#'.
    data = SudoersParser.COMMENTS_RE.sub("", data)
    return data

  def Parse(self, unused_stat, file_obj, unused_knowledge_base):
    self.ParseEntries(self._Preprocess(file_obj.read()))
    result = rdf_config_file.SudoersConfig()
    for entry in self.entries:
      # Handle multiple entries in one line, eg:
      # foo bar : baz
      # ... would become ...
      # [[foo, bar], [foo, baz]]
      key = entry[0]
      nested_entries = []
      if ":" not in entry:
        nested_entries = [entry]
      else:
        runner = []
        for field in entry:
          if field == ":":
            nested_entries.append(runner)
            runner = [key]
            continue

          runner.append(field)

        nested_entries.append(runner)

      for nested_entry in nested_entries:
        self._ParseSudoersEntry(nested_entry, result)

    yield result


text:
<?php

namespace Fly\Auth\Exception;

interface ExceptionInterface
{
}

redpajama_set_name: RedPajamaGithub


text: Varga is a Web Application for faculty at Sacred Heart Preparatory to manage substitute teachers.


text:
/* apps/s_apps.h */
/* Copyright (C) 1995-1998 Eric Young (eay@cryptsoft.com)
 * All rights reserved.
 *
 * This package is an SSL implementation written
 * by Eric Young (eay@cryptsoft.com).
 * The implementation was written so as to conform with Netscapes SSL.
 *
 * This library is free for commercial and non-commercial use as long as
 * the following conditions are aheared to.  The following conditions
 * apply to all code found in this distribution, be it the RC4, RSA,
 * lhash, DES, etc., code; not just the SSL code.  The SSL documentation
 * included with this distribution is covered by the same copyright terms
 * except that the holder is Tim Hudson (tjh@cryptsoft.com).
 *
 * Copyright remains Eric Young's, and as such any Copyright notices in
 * the code are not to be removed.
 * If this package is used in a product, Eric Young should be given attribution
 * as the author of the parts of the library used.
 * This can be in the form of a textual message at program startup or
 * in documentation (online or textual) provided with the package.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 * 1. Redistributions of source code must retain the copyright
 *    notice, this list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in the
 *    documentation and/or other materials provided with the distribution.
 * 3. All advertising materials mentioning features or use of this software
 *    must display the following acknowledgement:
 *    "This product includes cryptographic software written by
 *     Eric Young (eay@cryptsoft.com)"
 *    The word 'cryptographic' can be left out if the rouines from the library
 *    being used are not cryptographic related :-).
 * 4. If you include any Windows specific code (or a derivative thereof) from
 *    the apps directory (application code) you must include an acknowledgement:
 *    "This product includes software written by Tim Hudson (tjh@cryptsoft.com)"
 *
 * THIS SOFTWARE IS PROVIDED BY ERIC YOUNG ``AS IS'' AND
 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
 * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
 * SUCH DAMAGE.
 *
 * The licence and distribution terms for any publically available version or
 * derivative of this code cannot be changed.  i.e. this code cannot simply be
 * copied and put under another distribution licence
 * [including the GNU Public Licence.]
 */
/* ====================================================================
 * Copyright (c) 1998-2001 The OpenSSL Project.  All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 *
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 *
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in
 *    the documentation and/or other materials provided with the
 *    distribution.
 *
 * 3. All advertising materials mentioning features or use of this
 *    software must display the following acknowledgment:
 *    "This product includes software developed by the OpenSSL Project
 *    for use in the OpenSSL Toolkit. (http://www.openssl.org/)"
 *
 * 4. The names "OpenSSL Toolkit" and "OpenSSL Project" must not be used to
 *    endorse or promote products derived from this software without
 *    prior written permission. For written permission, please contact
 *    openssl-core@openssl.org.
 *
 * 5. Products derived from this software may not be called "OpenSSL"
 *    nor may "OpenSSL" appear in their names without prior written
 *    permission of the OpenSSL Project.
 *
 * 6. Redistributions of any form whatsoever must retain the following
 *    acknowledgment:
 *    "This product includes software developed by the OpenSSL Project
 *    for use in the OpenSSL Toolkit (http://www.openssl.org/)"
 *
 * THIS SOFTWARE IS PROVIDED BY THE OpenSSL PROJECT ``AS IS'' AND ANY
 * EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
 * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE OpenSSL PROJECT OR
 * ITS CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
 * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED
 * OF THE POSSIBILITY OF SUCH DAMAGE.
 * ====================================================================
 *
 * This product includes cryptographic software written by Eric Young
 * (eay@cryptsoft.com).  This product includes software written by Tim
 * Hudson (tjh@cryptsoft.com).
 *
 */
/* conflicts with winsock2 stuff on netware */
#if !defined(OPENSSL_SYS_NETWARE)
# include <sys/types.h>
#endif
#include <openssl/opensslconf.h>

#if defined(OPENSSL_SYS_WINDOWS) || defined(OPENSSL_SYS_MSDOS)
# include <conio.h>
#endif

#if defined(OPENSSL_SYS_MSDOS) && !defined(_WIN32)
# define _kbhit kbhit
#endif

#if defined(OPENSSL_SYS_VMS) && !defined(FD_SET)
/*
 * VAX C does not defined fd_set and friends, but it's actually quite simple
 */
/* These definitions are borrowed from SOCKETSHR.       /Richard Levitte */
# define MAX_NOFILE      32
# define NBBY             8     /* number of bits in a byte */

# ifndef FD_SETSIZE
#  define FD_SETSIZE      MAX_NOFILE
# endif                         /* FD_SETSIZE */

/* How many things we'll allow select to use. 0 if unlimited */
# define MAXSELFD        MAX_NOFILE
typedef int fd_mask;            /* int here! VMS prototypes int, not long */
# define NFDBITS (sizeof(fd_mask) * NBBY)/* bits per mask (power of 2!) */
# define NFDSHIFT 5             /* Shift based on above */

typedef fd_mask fd_set;
# define FD_SET(n, p)    (*(p) |= (1 << ((n) % NFDBITS)))
# define FD_CLR(n, p)    (*(p) &= ~(1 << ((n) % NFDBITS)))
# define FD_ISSET(n, p)  (*(p) & (1 << ((n) % NFDBITS)))
# define FD_ZERO(p)      memset((char *)(p), 0, sizeof(*(p)))
#endif

#define PORT            4433
#define PORT_STR        "4433"
#define PROTOCOL        "tcp"

int do_server(int port, int type, int *ret,
              int (*cb) (char *hostname, int s, int stype,
                         unsigned char *context), unsigned char *context,
              int naccept);
#ifdef HEADER_X509_H
int MS_CALLBACK verify_callback(int ok, X509_STORE_CTX *ctx);
#endif
#ifdef HEADER_SSL_H
int set_cert_stuff(SSL_CTX *ctx, char *cert_file, char *key_file);
int set_cert_key_stuff(SSL_CTX *ctx, X509 *cert, EVP_PKEY *key,
                       STACK_OF(X509) *chain, int build_chain);
int ssl_print_sigalgs(BIO *out, SSL *s);
int ssl_print_point_formats(BIO *out, SSL *s);
int ssl_print_curves(BIO *out, SSL *s, int noshared);
#endif
int ssl_print_tmp_key(BIO *out, SSL *s);
int init_client(int *sock, char *server, int port, int type);
int should_retry(int i);
int extract_port(char *str, short *port_ptr);
int extract_host_port(char *str, char **host_ptr, unsigned char *ip,
                      short *p);

long MS_CALLBACK bio_dump_callback(BIO *bio, int cmd, const char *argp,
                                   int argi, long argl, long ret);

#ifdef HEADER_SSL_H
void MS_CALLBACK apps_ssl_info_callback(const SSL *s, int where, int ret);
void MS_CALLBACK msg_cb(int write_p, int version, int content_type,
                        const void *buf, size_t len, SSL *ssl, void *arg);
void MS_CALLBACK tlsext_cb(SSL *s, int client_server, int type,
                           unsigned char *data, int len, void *arg);
#endif

int MS_CALLBACK generate_cookie_callback(SSL *ssl, unsigned char *cookie,
                                         unsigned int *cookie_len);
int MS_CALLBACK verify_cookie_callback(SSL *ssl, unsigned char *cookie,
                                       unsigned int cookie_len);

typedef struct ssl_excert_st SSL_EXCERT;

void ssl_ctx_set_excert(SSL_CTX *ctx, SSL_EXCERT *exc);
void ssl_excert_free(SSL_EXCERT *exc);
int args_excert(char ***pargs, int *pargc,
                int *badarg, BIO *err, SSL_EXCERT **pexc);
int load_excert(SSL_EXCERT **pexc, BIO *err);
void print_ssl_summary(BIO *bio, SSL *s);
#ifdef HEADER_SSL_H
int args_ssl(char ***pargs, int *pargc, SSL_CONF_CTX *cctx,
             int *badarg, BIO *err, STACK_OF(OPENSSL_STRING) **pstr);
int args_ssl_call(SSL_CTX *ctx, BIO *err, SSL_CONF_CTX *cctx,
                  STACK_OF(OPENSSL_STRING) *str, int no_ecdhe, int no_jpake);
int ssl_ctx_add_crls(SSL_CTX *ctx, STACK_OF(X509_CRL) *crls,
                     int crl_download);
int ssl_load_stores(SSL_CTX *ctx, const char *vfyCApath,
                    const char *vfyCAfile, const char *chCApath,
                    const char *chCAfile, STACK_OF(X509_CRL) *crls,
                    int crl_download);
#endif


text:
<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN"
                      "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd">

    <meta http-equiv="content-type" content="text/html; charset=UTF-8"/>
    <title>Building Indexes - Zend Framework Manual</title>

                    Overview

Zend_Search_Lucene

                        Programmer's Reference Guide

Searching an Index

Building Indexes

Creating a New Index

            Index creation and updating capabilities are implemented within the
            Zend_Search_Lucene component, as well as the Java Lucene project.
            You can use either of these options to create indexes that
            Zend_Search_Lucene can search.

            The <acronym class="acronym">PHP</acronym> code listing below provides an example of how to index a file
            using Zend_Search_Lucene indexing <acronym class="acronym">API</acronym>:

// Create index

$index = Zend_Search_Lucene::create('/data/my-index');

$doc = new Zend_Search_Lucene_Document();

// Store document URL to identify it in the search results

$doc->addField(Zend_Search_Lucene_Field::Text('url', $docUrl));

// Index document contents

$doc->addField(Zend_Search_Lucene_Field::UnStored('contents', $docContent));

// Add document to the index

$index->addDocument($doc);

            Newly added documents are immediately searchable in the index.

Updating Index

            The same procedure is used to update an existing index. The only difference
            is that the open() method is called instead of the create() method:

// Open existing index

$index = Zend_Search_Lucene::open('/data/my-index');

$doc = new Zend_Search_Lucene_Document();

// Store document URL to identify it in search result.

$doc->addField(Zend_Search_Lucene_Field::Text('url', $docUrl));

// Index document content

$doc->addField(Zend_Search_Lucene_Field::UnStored('contents',

                                                  $docContent));

// Add document to the index.

$index->addDocument($doc);

Updating Documents

            The Lucene index file format doesn't support document updating.
            Documents should be removed and re-added to the index to effectively update them.

             Zend_Search_Lucene::delete() method operates with an internal
            index document id. It can be retrieved from a query hit by 'id' property:

$removePath = ...;

$hits = $index->find('path:' . $removePath);

foreach ($hits as $hit) {

    $index->delete($hit->id);

}

Retrieving Index Size

            There are two methods to retrieve the size of an index in
            Zend_Search_Lucene.

              Zend_Search_Lucene::maxDoc() returns one greater than the
             largest possible document number. It's actually the overall number of the documents in
             the index including deleted documents, so it has a synonym:
              Zend_Search_Lucene::count().

              Zend_Search_Lucene::numDocs() returns the total number of
             non-deleted documents.

$indexSize = $index->count();

$documents = $index->numDocs();

             Zend_Search_Lucene::isDeleted($id) method may be used to check
            if a document is deleted.

for ($count = 0; $count < $index->maxDoc(); $count++) {

    if ($index->isDeleted($count)) {

        echo "Document #$id is deleted.\n";

    }

}

            Index optimization removes deleted documents and squeezes documents' IDs in to a smaller
            range. A document's internal id may therefore change during index optimization.

Index optimization

            A Lucene index consists of many segments. Each segment is a completely independent set
            of data.

            Lucene index segment files can't be updated by design. A segment update needs full
            segment reorganization. See Lucene index file formats for details (» http://lucene.apache.org/java/2_3_0/fileformats.html)

            [1]

            .

            New documents are added to the index by creating new segment.

            Increasing number of segments reduces quality of the index, but index optimization
            restores it. Optimization essentially merges several segments into a new one. This
            process also doesn't update segments. It generates one new large segment and updates
            segment list ('segments' file).

            Full index optimization can be trigger by calling the
             Zend_Search_Lucene::optimize() method. It merges all index
            segments into one new segment:

// Open existing index

$index = Zend_Search_Lucene::open('/data/my-index');

// Optimize index.

$index->optimize();

            Automatic index optimization is performed to keep indexes in a consistent state.

            Automatic optimization is an iterative process managed by several index options. It
            merges very small segments into larger ones, then merges these larger segments into even
            larger segments and so on.

MaxBufferedDocs auto-optimization option

                MaxBufferedDocs is a minimal number of documents required
                before the buffered in-memory documents are written into a new segment.

                MaxBufferedDocs can be retrieved or set by
                $index->getMaxBufferedDocs() or
                $index->setMaxBufferedDocs($maxBufferedDocs) calls.

                Default value is 10.

MaxMergeDocs auto-optimization option

                MaxMergeDocs is a largest number of documents ever merged by
                addDocument(). Small values (e.g., less than 10.000) are best for interactive
                indexing, as this limits the length of pauses while indexing to a few seconds.
                Larger values are best for batched indexing and speedier searches.

                MaxMergeDocs can be retrieved or set by
                $index->getMaxMergeDocs() or
                $index->setMaxMergeDocs($maxMergeDocs) calls.

                Default value is PHP_INT_MAX.

MergeFactor auto-optimization option

                MergeFactor determines how often segment indices are merged by
                addDocument(). With smaller values, less <acronym class="acronym">RAM</acronym> is used while
                indexing, and searches on unoptimized indices are faster, but indexing speed is
                slower. With larger values, more <acronym class="acronym">RAM</acronym> is used during indexing, and
                while searches on unoptimized indices are slower, indexing is faster. Thus larger
                values (> 10) are best for batch index creation, and smaller values (< 10) for
                indices that are interactively maintained.

                MergeFactor is a good estimation for average number of segments
                merged by one auto-optimization pass. Too large values produce large number of
                segments while they are not merged into new one. It may be a cause of "failed to
                open stream: Too many open files" error message. This limitation is system
                dependent.

                MergeFactor can be retrieved or set by
                $index->getMergeFactor() or
                $index->setMergeFactor($mergeFactor) calls.

                Default value is 10.

                Lucene Java and Luke (Lucene Index Toolbox - » http://www.getopt.org/luke/) can also
                be used to optimize an index. Latest Luke release (v0.8) is based on Lucene v2.3 and
                compatible with current implementation of Zend_Search_Lucene
                component (Zend Framework 1.6). Earlier versions of
                Zend_Search_Lucene implementations need another versions of
                Java Lucene tools to be compatible:

                            Zend Framework 1.5 - Java Lucene 2.1 (Luke tool v0.7.1 - <tt class="uri">http://www.getopt.org/luke/luke-0.7.1/</tt>)

                            Zend Framework 1.0 - Java Lucene 1.4 - 2.1 (Luke tool v0.6 - <tt class="uri">http://www.getopt.org/luke/luke-0.6/</tt>)

Permissions

            By default, index files are available for reading and writing by everyone.

            It's possible to override this with the
             Zend_Search_Lucene_Storage_Directory_Filesystem::setDefaultFilePermissions()
            method:

// Get current default file permissions

$currentPermissions =

    Zend_Search_Lucene_Storage_Directory_Filesystem::getDefaultFilePermissions();

// Give read-writing permissions only for current user and group

Zend_Search_Lucene_Storage_Directory_Filesystem::setDefaultFilePermissions(0660);

Limitations

Index size

                Index size is limited by 2GB for 32-bit platforms.

                Use 64-bit platforms for larger indices.

Supported Filesystems

                Zend_Search_Lucene uses  flock() to
                provide concurrent searching, index updating and optimization.

                According to the <acronym class="acronym">PHP</acronym> » documentation,
                " flock() will not work on NFS and many other networked file
                systems".

                Do not use networked file systems with Zend_Search_Lucene.

[1]
                    The currently supported Lucene index file format is version 2.3 (starting from
                    Zend Framework 1.6).

                    Overview

Zend_Search_Lucene

                        Programmer's Reference Guide

Searching an Index

Programmer's Reference Guide

Programmer's Reference Guide

Zend Framework Reference

Zend_Search_Lucene

Overview

Building Indexes

Searching an Index

Query Language

Query Construction API

Character Set

Extensibility

Interoperating with Java Lucene

Advanced

Best Practices


text:
from __future__ import absolute_import

# imports - airdraw
from airdraw.config.base import BaseConfig
from airdraw.config.app  import AppConfig


text:
package util

import (
	"context"

	api "github.com/polydawn/go-timeless-api"
	"github.com/polydawn/go-timeless-api/rio"
	"github.com/polydawn/rio/fs"
	nilFS "github.com/polydawn/rio/fs/nilfs"
	"github.com/polydawn/rio/fs/osfs"
	. "github.com/warpfork/go-errcat"
)

// A "scan" is roughly the same as an unpack to /dev/null,
// but takes a single URL, and *doesn't* require a hash.
//
// It can even populate the CAS cache!
//
// However, note that it's an overall intention to make this feature
// usable only very knowingly and with moderate inconvenience -- because you
// *should not* do it in the middle of a script; you should be doing any scans
// *once* and then tracking the resulting references via a release catalog:
// which keeps the overall process more controlled, auditable, and
// well-defined even in the case of untrusted networks.

func CreateScanner(t api.PackType, unpacker unpackFn) rio.ScanFunc {
	return func(
		ctx context.Context, // Long-running call.  Cancellable.
		packType api.PackType, // The name of pack format.
		filt api.FilesetUnpackFilter, // Optionally: filters we should apply while unpacking.
		placementMode rio.PlacementMode, // For scanning only "None" (cache; the default) and "Direct" (don't cache) are valid.
		addr api.WarehouseLocation, // The *one* warehouse to fetch from.  Must be a monowarehouse (not a CA-mode).
		mon rio.Monitor, // Optionally: callbacks for progress monitoring.
	) (_ api.WareID, err error) {
		if mon.Chan != nil {
			defer close(mon.Chan)
		}
		defer RequireErrorHasCategory(&err, rio.ErrorCategory(""))

		// Sanitize arguments.
		if packType != t {
			return api.WareID{}, Errorf(rio.ErrUsage, "this transmat implementation only supports packtype %q (not %q)", t, packType)
		}
		if !filt.IsComplete() {
			return api.WareID{}, Errorf(rio.ErrUsage, "filters must be completely specified")
		}
		if placementMode == "" {
			placementMode = rio.Placement_None
		}

		// TODO FUTURE actually support cache

		// Dial warehouse.
		//  Note how this is a subset of the usual accepted warehouses;
		//  it must be a monowarehouse, not a legit CA storage bucket.
		reader, err := PickReader(api.WareID{t, "-"}, []api.WarehouseLocation{addr}, true, mon)
		if err != nil {
			return api.WareID{}, err
		}
		defer reader.Close()

		// Construct filesystem wrapper to use for all our ops.
		//  If caching, it's a real fs handle;
		//  if not, it's a bunch of no-op'ing functions.
		var afs fs.FS
		switch placementMode {
		case rio.Placement_None:
			afs = osfs.New(fs.MustAbsolutePath("/nope/nope")) // TODO cache
		case rio.Placement_Direct:
			afs = nilFS.New()
		default:
			panic("unreachable")

		}

		// Extract.
		//  For once we can actually discard the *prefilter* wareID, since we don't have
		//  an expected one to assert against.
		//  TODO: the ware used by the buffer internally will need to be derived from addr
		//  once caching is supported.
		_, unpackedWareID, err := unpacker(ctx, afs, filt, api.WareID{t, "-"}, reader, mon)
		return unpackedWareID, err
	}
}


text:
package org.jclouds.ultradns.ws.parse;

import static org.testng.Assert.assertEquals;

import java.io.InputStream;

import org.jclouds.http.functions.BaseHandlerTest;
import org.jclouds.ultradns.ws.domain.Zone;
import org.jclouds.ultradns.ws.domain.Zone.DNSSECStatus;
import org.jclouds.ultradns.ws.xml.ZoneListHandler;
import org.testng.annotations.Test;

import com.google.common.collect.FluentIterable;
import com.google.common.collect.ImmutableSet;

@Test(testName = "GetZonesOfAccountResponseTest")
public class GetZonesOfAccountResponseTest extends BaseHandlerTest {

   public void test() {
      InputStream is = getClass().getResourceAsStream("/zones.xml");

      FluentIterable<Zone> expected = expected();

      ZoneListHandler handler = injector.getInstance(ZoneListHandler.class);
      FluentIterable<Zone> result = factory.create(handler).parse(is);

      assertEquals(result.toSet().toString(), expected.toSet().toString());
   }

   public FluentIterable<Zone> expected() {
      return FluentIterable.from(ImmutableSet.<Zone> builder()
                           .add(Zone.builder()
                                    .name("jclouds.org.")
                                    .typeCode(1)
                                    .accountId("AAAAAAAAAAAAAAAA")
                                    .ownerId("EEEEEEEEEEEEEEEE")
                                    .id("0000000000000001")
                                    .dnssecStatus(DNSSECStatus.UNSIGNED).build())
                           .add(Zone.builder()
                                    .name("0.1.2.3.4.5.6.7.ip6.arpa.")
                                    .typeCode(1)
                                    .accountId("AAAAAAAAAAAAAAAA")
                                    .ownerId("EEEEEEEEEEEEEEEE")
                                    .id("0000000000000002")
                                    .dnssecStatus(DNSSECStatus.UNSIGNED).build())
                           .build());
   }

}


text:
package io.digdag.core.session;

import java.time.Instant;
import com.fasterxml.jackson.databind.annotation.JsonSerialize;
import com.fasterxml.jackson.databind.annotation.JsonDeserialize;
import org.immutables.value.Value;

@JsonDeserialize(as = ImmutableStoredSessionAttempt.class)
public abstract class StoredSessionAttempt
        extends SessionAttempt
{
    public abstract long getId();

    public abstract AttemptStateFlags getStateFlags();

    public abstract long getSessionId();

    public abstract Instant getCreatedAt();

    public static StoredSessionAttempt copyOf(StoredSessionAttempt o) {
        return ImmutableStoredSessionAttempt.builder().from(o).build();
    }
}


text:
'use strict'

const fs = require('fs')
const path = require('path')
const expect = require('chai').expect
const CKIPResponse = require('../lib/ckip_response')

describe('CKIPResponse', () => {
  describe('#constructor()', () => {
    it('should be instantiated', () => {
      const data = fs.readFileSync(path.resolve(__dirname, './fixtures/response_success.xml'))
      const response = new CKIPResponse(data)
      expect(response).to.be.an.instanceof(CKIPResponse)
      expect(response.data).to.be.an.instanceof(Buffer)
      expect(response.data.toString()).to.equal(data.toString())
    })
  })

  describe('#segment()', () => {
    it('should segment', async () => {
      const data = fs.readFileSync(path.resolve(__dirname, './fixtures/response_success.xml'))
      const response = new CKIPResponse(data)
      const sentences = await response.segment()
      const words = sentences[0]
      expect(sentences).to.have.length(1)
      expect(words).to.have.members(['台新', '金控', '12月', '3日', '將', '召開', '股東', '臨時會', '進行', '董監', '改選', '。'])
    })

    it('should segment with POS tagging', async () => {
      const data = fs.readFileSync(path.resolve(__dirname, './fixtures/response_success.xml'))
      const response = new CKIPResponse(data)
      const sentences = await response.segment({ tag: true })
      const words = sentences[0]
      expect(sentences).to.have.length(1)
      expect(words).to.have.deep.members([
        { word: '台新', tag: 'N' },
        { word: '金控', tag: 'N' },
        { word: '12月', tag: 'N' },
        { word: '3日', tag: 'N' },
        { word: '將', tag: 'ADV' },
        { word: '召開', tag: 'Vt' },
        { word: '股東', tag: 'N' },
        { word: '臨時會', tag: 'N' },
        { word: '進行', tag: 'Vt' },
        { word: '董監', tag: 'N' },
        { word: '改選', tag: 'Nv' },
        { word: '。', tag: 'PERIODCATEGORY' }
      ])
    })

    it('should throw error if authentication failed', async () => {
      const data = fs.readFileSync(path.resolve(__dirname, './fixtures/response_error.xml'))
      const response = new CKIPResponse(data)
      try {
        await response.segment()
      } catch (err) {
        expect(err).to.be.an('error')
      }
    })
  })
})


text:
<?xml version="1.0" encoding="UTF-8" ?>
<resources>
	<!-- -->
	<resource name="TagList">Tags</resource>
	<resource name="Normal">Normal</resource>
	<resource name="Definition Term">Definition Term</resource>
	<resource name="MenuList">Menu List</resource>
	<resource name="Heading 1">Heading 1</resource>
	<resource name="Heading 2">Heading 2</resource>
	<resource name="Heading 3">Heading 3</resource>
	<resource name="Heading 4">Heading 4</resource>
	<resource name="Heading 5">Heading 5</resource>
	<resource name="Heading 6">Heading 6</resource>
	<!-- Buttons -->
	<resource name="Save">Spara</resource>
	<resource name="New">Ny</resource>
	<resource name="Print">Skriv ut</resource>
	<resource name="Cut">Klipp ut</resource>
	<resource name="Copy">Kopiera</resource>
	<resource name="Paste">Klistra in</resource>
	<resource name="PasteText">Klistra in ren text</resource>
	<resource name="PasteWord">Klistra in från Word</resource>
	<resource name="Delete">Radera</resource>
	<resource name="Undo">Ångra</resource>
	<resource name="Redo">Gör om</resource>
	<resource name="SelectAll">Markera alla</resource>
	<resource name="ToggleBorder">Synliga kantlinjer</resource>
	<resource name="AbsolutePosition">Absolut positionering</resource>
	<resource name="InsertHorizontalRule">Infoga linje</resource>
	<resource name="InsertLink">Infoga länk</resource>
	<resource name="Unlink">Ta bort länk</resource>
	<resource name="InsertDocument">Nedladdningsbara filer</resource>
	<resource name="InsertChars">Specialtecken</resource>
	<resource name="InsertEmotion">Infoga smilies</resource>
	<resource name="InsertText">Infoga, ändra eller repositionera textstycken</resource>
	<resource name="InsertImage">Infoga lokal bild</resource>
	<resource name="InsertFlash">Infoga Flash-animation</resource>
	<resource name="BringForward">Skicka framåt</resource>
	<resource name="BringBackward">Skicka bakåt</resource>
	<resource name="TableDropDown">Infoga/redigera tabell</resource>
	<resource name="InsertRow">Infoga rader</resource>
	<resource name="DeleteRow">Radera rader</resource>
	<resource name="InsertColumn">Infoga kolumner</resource>
	<resource name="DeleteColumn">Radera kolumner</resource>
	<resource name="InsertCell">Infoga celler</resource>
	<resource name="DeleteCell">Radera celler</resource>
	<resource name="MergeCell">Sammanfoga celler</resource>
	<resource name="SplitCell">Dela upp celler</resource>
	<resource name="Bold">Fet stil</resource>
	<resource name="Italic">Kursiv stil</resource>
	<resource name="Underline">Understruken text</resource>
	<resource name="JustifyLeft">Vänsterjustera</resource>
	<resource name="JustifyCenter">Centrera</resource>
	<resource name="JustifyRight">Högerjustera</resource>
	<resource name="ForeColor">Textfärg</resource>
	<resource name="SetForeColor">Välj textfärg</resource>
	<resource name="BackColor">Bakgrund</resource>
	<resource name="SetBackColor">Välj bakgrundsfärg</resource>
	<resource name="Automatic">Automatic</resource>
	<resource name="MoreColors">More Colors...</resource>
	<resource name="InsertOrderedList">Infoga nummerlista</resource>
	<resource name="InsertUnorderedList">Infoga punktlista</resource>
	<resource name="Indent">Gör indrag på text</resource>
	<resource name="Outdent">Gör utdrag på text</resource>
	<resource name="Subscript">Nedsänkt text</resource>
	<resource name="Superscript">Upphöjd text</resource>
	<resource name="Strikethrough">Överstruken text</resource>
	<resource name="RemoveFormat">Ta bort formatering</resource>
	<resource name="CleanCode">Kodupprensning</resource>
	<resource name="SelectNone">Markera ingen</resource>
	<resource name="JustifyNone">Ingen styckejustering</resource>
	<resource name="JustifyFull">Marginaljustera</resource>
	<resource name="InsertDate">Infoga dagens datum</resource>
	<resource name="InsertTime">Infoga aktuellt klockslag</resource>
	<resource name="Help">Hjälp</resource>
	<resource name="Paragraph">Stycke</resource>
	<resource name="InsertParagraph">Stycke</resource>
	<resource name="FontName">Typsnitt</resource>
	<resource name="FontSize">Storlek</resource>
	<resource name="Zoom">Zooma</resource>
	<resource name="CssClass">Css Class</resource>
	<resource name="CssStyle">Inline Stil</resource>
	<resource name="Links">Länkar</resource>
	<resource name="Codes">Kodsnutt</resource>
	<resource name="ToFullPage">Fit to Window</resource>
	<resource name="FromFullPage">Restore down</resource>
	<resource name="imagesDropdown">Bilder</resource>
	<resource name="DocumentPropertyPage">Page Properties</resource>
	<resource name="Normal">Normal text</resource>
	<resource name="HTML">HTML</resource>
	<resource name="Preview">Förhandsgranska</resource>
	<resource name="Cancel">Avbryt</resource>
	<resource name="Properties">Properties</resource>
	<resource name="Width">Bredd</resource>
	<resource name="Height">Höjd</resource>
	<resource name="Transparency">Transparency</resource>
	<resource name="Quality">Quality</resource>
	<resource name="Upload">Upload</resource>
	<resource name="Insert">Insert</resource>
	<resource name="Layout">Layout</resource>
	<resource name="Alignment">Justering</resource>
	<resource name="Border_Color">Kantlinjefärg</resource>
	<resource name="Spacing">Spacing</resource>
	<resource name="Horizontal">Horizontal</resource>
	<resource name="Vertical">Vertical</resource>
	<resource name="Alternate">Alternate Text</resource>
	<resource name="Hyperlink_Information">Hyperlink Information</resource>
	<resource name="Title">Title</resource>
	<resource name="Target">Target</resource>
	<resource name="Link_Color">Link Color</resource>
	<resource name="Rows">Rows</resource>
	<resource name="Columns">Columns</resource>
	<resource name="TableAttribute">Table attributes</resource>
	<resource name="Pixels">Pixels</resource>
	<resource name="Percent">Percent</resource>
	<resource name="NotSet">NotSet</resource>
	<resource name="Left">Left</resource>
	<resource name="Right">Right</resource>
	<resource name="Texttop">Texttop</resource>
	<resource name="Absmiddle">Absmiddle</resource>
	<resource name="Baseline">Baseline</resource>
	<resource name="Absbottom">Absbottom</resource>
	<resource name="Bottom">Bottom</resource>
	<resource name="Middle">Middle</resource>
	<resource name="Top">Top</resource>
	<resource name="Bordersize">Kantlinjestorlek</resource>
	<resource name="Backgroundcolor">Bakgrundsfärg</resource>
	<resource name="CellSpacing">Cellavstånd</resource>
	<resource name="BorderColor">Kantlinjefärg</resource>
	<resource name="CellPadding">Cellutökning</resource>
	<resource name="OK">OK</resource>
	<resource name="EditRow">Row Properties</resource>
	<resource name="EditCell">Cell Properties</resource>
  <resource name="CustomBullet">Custom bullet</resource>
	<resource name="InsertTemplate">Insert Templates</resource>
	<resource name="Description">Description</resource>
	<resource name="Keywords">Keywords</resource>
	<resource name="HtmlBase">Base location</resource>
	<resource name="InsertAnchor">Infoga Anchor</resource>
	<resource name="InsertMedia">Infoga Media</resource>
	<resource name="ImageGalleryByBrowsing">Infoga lokal bild</resource>
	<resource name="Type">filtyper</resource>
	<resource name="Size">Storlek</resource>
	<resource name="All">Alla</resource>
	<resource name="Send">Skicka</resource>
	<resource name="uploadResult">Det du laddat upp har sparats som en fil</resource>
	<resource name="Createddate">Skapat datum</resource>
	<resource name="Modifieddate">Ändrat datum:</resource>
	<resource name="Next">Nästa</resource>
	<resource name="Prev">Prev</resource>
	<resource name="Total">Totalt</resource>
	<resource name="Images">bilder</resource>
	<resource name="Name">Namn</resource>
	<resource name="NotSet">Inte inställt</resource>
	<resource name="Center">Centrera</resource>
	<resource name="Attributes">Attributes</resource>
	<resource name="Newwindow">New window</resource>
	<resource name="Parentwindow">Parent window</resource>
	<resource name="Samewindow">Same window</resource>
	<resource name="Topmostwindow">Topmost window</resource>
	<resource name="AutoStart">AutoStart</resource>
	<resource name="ShowControls">ShowControls</resource>
	<resource name="ShowStatusBar">ShowStatusBar</resource>
	<resource name="Edit">Edit</resource>
	<resource name="Formatting">Formatting</resource>
	<resource name="Table">Table</resource>
	<resource name="Justify">Justify</resource>
	<resource name="Font">Font</resource>
	<resource name="Text">Text</resource>
	<resource name="Background">Background</resource>
	<resource name="Border">Borders</resource>
	<resource name="Layout">Layout</resource>
	<resource name="List">List</resource>
	<resource name="Other">Other</resource>
	<resource name="Cursor">Cursor</resource>
	<resource name="Filter">Filter</resource>
	<resource name="Demo">Demo</resource>
	<resource name="DemoText">AsBbYsGgLlJj</resource>
	<resource name="Color">Color</resource>
	<resource name="Url">Url</resource>
	<!-- Paragraph DropDown-->
	<resource name="Heading1">Rubrik 1</resource>
	<resource name="Heading2">Rubrik 2</resource>
	<resource name="Heading3">Rubrik 3</resource>
	<resource name="Heading4">Rubrik 4</resource>
	<resource name="Heading5">Rubrik 5</resource>
	<resource name="Heading6">Rubrik 6</resource>
	<resource name="Address">Adress</resource>
	<resource name="Formatted">Formatterad</resource>
	<resource name="Definition Term">Definitionsterm</resource>
	<!-- Buttons Image URL -->
	<resource name="BoldImage">bold.gif</resource>
	<resource name="UnderImage">under.gif</resource>
	<resource name="PreviewImage">preview.gif</resource>
	<resource name="DesignImage">design.gif</resource>
	<resource name="HtmlImage">htmlview.gif</resource>
	<resource name="RemoveWordTags">Do you want to clean the code before pasting from Word?</resource>
	<resource name="RemoveHTMLTags">Remove all HTML tags</resource>
	<resource name="RemoveWordMarkup">Remove all word specific markup</resource>
	<resource name="CleanupCSS">Clean up cascading style sheets</resource>
	<resource name="CleanupFont"><![CDATA[Clean up <Font> tags]]></resource>
	<resource name="CleanupSpan"><![CDATA[Clean up <Span> tags]]></resource>

	<!-- version 6.0 -->
	<resource name="Sort">Sort</resource>
	<resource name="SortByDateDesc">Date Desc</resource>
	<resource name="SortByDateAsc">Date Asc</resource>
	<resource name="SortBySizeDesc">Size Desc</resource>
	<resource name="SortBySizeAsc">Size Asc</resource>
	<resource name="ActiveXObject">Object</resource>
	<resource name="noWrap">noWrap</resource>
	<resource name="CellScope">Cell Scope</resource>
	<resource name="PasteAsHTML">Paste As HTML</resource>
	<resource name="ImageFileIsTooLarge">The uploaded image is too large. The max image file size of this web site is set to: </resource>
	<resource name="ConstrainProportions">Constrain Proportions</resource>
	<resource name="MaxFileSizeAllowed">Max file size allowed</resource>
	<resource name="MaxImagedImension">Max image dimension is</resource>
	<resource name="pixels">pixels</resource>
	<resource name="AutomaticImageResizeOnOff">Auto resize uploaded images function is</resource>
	<resource name="on">on</resource>
	<resource name="off">off</resource>
	<resource name="MaxFolderSizeAllowed">Max Upload folder size is </resource>
	<resource name="Used">Used</resource>
	<resource name="WrongFormatImage">The Image Format is Wrong!</resource>
	<resource name="TemplateCreated">Template created successfully!</resource>
	<resource name="TemplateModified">Template modified successfully!</resource>
	<resource name="InputRequired">Input Required:</resource>
	<resource name="1ColumnTable">One-Column Table</resource>
	<resource name="2ColumnTable">Two-Column Table</resource>
	<resource name="3ColumnTable">Three-Column Table</resource>
	<resource name="Header-Right-TopLeft-Body">Header, Right, Top Left and Body</resource>
	<resource name="Header-Left-TopRight-Body">Header, Left, Top Right and Body</resource>
	<resource name="Header-Footer-3-Columns">Header, Footer and 3 Columns</resource>
	<resource name="Uploading">Uploading...</resource>
	<resource name="maxRequestLengthExceeded"><![CDATA[Maximum request length exceeded.The default value for the maxRequestLength parameter in the <httpRuntime> section of the Web.Config file is 4096 (4 MB). As a result, files that are larger than this value are not uploaded by default. ]]></resource>
	<resource name="Blue">Blue</resource>
	<resource name="Purple">Purple</resource>
	<resource name="Red">Red</resource>
	<resource name="Orange">Orange</resource>
	<resource name="Green">Green</resource>
	<resource name="Cyan">Cyan</resource>
	<resource name="White">White</resource>
	<resource name="Black">Black</resource>
	<resource name="WordNotFound">Word not found. Nothing replaced.</resource>
	<resource name="WordReplaced">Word(s) were replaced</resource>
	<resource name="DIV">DIV</resource>
	<resource name="WebPalette">Web Palette</resource>
	<resource name="NamedColors">Named Colors </resource>
	<resource name="CustomColor">Custom Color</resource>
	<resource name="Characters">Characters</resource>
	<resource name="Words">Words</resource>
	<resource name="Close">Close</resource>
	<resource name="EditorEmpty"><![CDATA[\nEditor is empty.\n\nNothing to clean up.]]></resource>
	<resource name="Square">Square</resource>
	<resource name="Circle">Circle</resource>
	<resource name="StartingNumber">Starting Number</resource>
	<resource name="SelectColor">Select a color</resource>
	<resource name="Recent">Recent</resource>
	<resource name="Loading">Loading...</resource>
	<resource name="FormatBlock">Normal</resource>
</resources>


text:
importScripts = [
    '../js/lib.js',
    '../js/ItemSpec.js',
    '../js/FormManager.js',
    '../js/Uploaders.js'
];
importScripts.forEach(function(val,index,arr) {
    $("head").append(
        '<script type="text/javascript" src="' + val + '"></script>'
    );
});

// ======================================================================
// LostForm

function LostForm(uploader) {
    if ( !( uploader && "isA" in uploader && uploader.isA(Uploader) ) )
        alert('LostForm has to have an Uploader!');
    this.uploader = uploader;

    this.currItem = new ItemSpec(
        $("#lostname"),
        $("#item"),
        $("#lostemail"),
        $("#lostdate"),
        $("#phone"),
        $("#lastloc"),
        $("#img"),
        $("#itemdesc")
    );
    // if focus is REMOVED from a text input field, remove the notification of
    // invalidness until the next submission
    this.currItem.forEach(function(field) {
        field.obj.focusout(function() {field.obj.removeClass('notValid');});
    });
}
extend(LostForm,FormManager);

// --------------------------------------------------
// upload item to db

LostForm.prototype.upload = function() {
    console.log('entered FormManager.upload');
    this.uploader.upload(this.currItem,PARSEUPLOAD_CODE_LOST);
    console.log('left FormManager.upload');
};

// function to hide and show the input box if item name is other than the listed
$(function() {
    //initially hide the textbox
    $("#othername").hide();
    $('#item').change(function() {
        if($(this).find('option:selected').val() == "Other"){
            $("#othername").show();
        } else {
            $("#othername").hide();
        }
    });
});

// ==================================================
// main function

var main = function() {
    var uploader = new ParseUploader();
    var lostForm = new LostForm(uploader);
    $('#submit').click(function() {lostForm.callback()});
}

$(document).ready(main);