text:
#include <config.h>

#include <stdio.h>
#include <stdlib.h>
#ifdef HAVE_STRING_H
# include <string.h>
#else
# include <strings.h>
#endif
#ifdef HAVE_UNISTD_H
# include <unistd.h>
#endif
#include <sys/types.h>
#include <assert.h>
#include <errno.h>

#include "wget.h"
#include "utils.h"
#include "url.h"
#include "retr.h"
#include "ftp.h"
#include "connect.h"
#include "host.h"
#include "netrc.h"
#include "convert.h"		/* for downloaded_file */
#include "recur.h"		/* for INFINITE_RECURSION */

#ifndef errno
extern int errno;
#endif

extern SUM_SIZE_INT total_downloaded_bytes;

/* File where the "ls -al" listing will be saved.  */
#define LIST_FILENAME ".listing"

extern char ftp_last_respline[];

extern FILE *output_stream;
extern int output_stream_regular;

typedef struct
{
  int st;			/* connection status */
  int cmd;			/* command code */
  int csock;			/* control connection socket */
  double dltime;		/* time of the download in msecs */
  enum stype rs;		/* remote system reported by ftp server */
  char *id;			/* initial directory */
  char *target;			/* target file name */
  struct url *proxy;		/* FTWK-style proxy */
} ccon;

/* Look for regexp "( *[0-9]+ *byte" (literal parenthesis) anywhere in
   the string S, and return the number converted to wgint, if found, 0
   otherwise.  */
static wgint
ftp_expected_bytes (const char *s)
{
  wgint res;

  while (1)
    {
      while (*s && *s != '(')
	++s;
      if (!*s)
	return 0;
      ++s;			/* skip the '(' */
      res = str_to_wgint (s, (char **) &s, 10);
      if (!*s)
	return 0;
      while (*s && ISSPACE (*s))
	++s;
      if (!*s)
	return 0;
      if (TOLOWER (*s) != 'b')
	continue;
      if (strncasecmp (s, "byte", 4))
	continue;
      else
	break;
    }
  return res;
}

#ifdef ENABLE_IPV6
/*
 * This function sets up a passive data connection with the FTP server.
 * It is merely a wrapper around ftp_epsv, ftp_lpsv and ftp_pasv.
 */
static uerr_t
ftp_do_pasv (int csock, ip_address *addr, int *port)
{
  uerr_t err;

  /* We need to determine the address family and need to call
     getpeername, so while we're at it, store the address to ADDR.
     ftp_pasv and ftp_lpsv can simply override it.  */
  if (!socket_ip_address (csock, addr, ENDPOINT_PEER))
    abort ();

  /* If our control connection is over IPv6, then we first try EPSV and then
   * LPSV if the former is not supported. If the control connection is over
   * IPv4, we simply issue the good old PASV request. */
  switch (addr->type)
    {
    case IPV4_ADDRESS:
      if (!opt.server_response)
        logputs (LOG_VERBOSE, "==> PASV ... ");
      err = ftp_pasv (csock, addr, port);
      break;
    case IPV6_ADDRESS:
      if (!opt.server_response)
        logputs (LOG_VERBOSE, "==> EPSV ... ");
      err = ftp_epsv (csock, addr, port);

      /* If EPSV is not supported try LPSV */
      if (err == FTPNOPASV)
        {
          if (!opt.server_response)
            logputs (LOG_VERBOSE, "==> LPSV ... ");
          err = ftp_lpsv (csock, addr, port);
        }
      break;
    default:
      abort ();
    }

  return err;
}

/*
 * This function sets up an active data connection with the FTP server.
 * It is merely a wrapper around ftp_eprt, ftp_lprt and ftp_port.
 */
static uerr_t
ftp_do_port (int csock, int *local_sock)
{
  uerr_t err;
  ip_address cip;

  if (!socket_ip_address (csock, &cip, ENDPOINT_PEER))
    abort ();

  /* If our control connection is over IPv6, then we first try EPRT and then
   * LPRT if the former is not supported. If the control connection is over
   * IPv4, we simply issue the good old PORT request. */
  switch (cip.type)
    {
    case IPV4_ADDRESS:
      if (!opt.server_response)
        logputs (LOG_VERBOSE, "==> PORT ... ");
      err = ftp_port (csock, local_sock);
      break;
    case IPV6_ADDRESS:
      if (!opt.server_response)
        logputs (LOG_VERBOSE, "==> EPRT ... ");
      err = ftp_eprt (csock, local_sock);

      /* If EPRT is not supported try LPRT */
      if (err == FTPPORTERR)
        {
          if (!opt.server_response)
            logputs (LOG_VERBOSE, "==> LPRT ... ");
          err = ftp_lprt (csock, local_sock);
        }
      break;
    default:
      abort ();
    }
  return err;
}
#else

static uerr_t
ftp_do_pasv (int csock, ip_address *addr, int *port)
{
  if (!opt.server_response)
    logputs (LOG_VERBOSE, "==> PASV ... ");
  return ftp_pasv (csock, addr, port);
}

static uerr_t
ftp_do_port (int csock, int *local_sock)
{
  if (!opt.server_response)
    logputs (LOG_VERBOSE, "==> PORT ... ");
  return ftp_port (csock, local_sock);
}
#endif

static void
print_length (wgint size, wgint start, int authoritative)
{
  logprintf (LOG_VERBOSE, _("Length: %s"), with_thousand_seps (size));
  if (size >= 1024)
    logprintf (LOG_VERBOSE, " (%s)", human_readable (size));
  if (start > 0)
    {
      if (start >= 1024)
	logprintf (LOG_VERBOSE, _(", %s (%s) remaining"),
		   with_thousand_seps (size - start),
		   human_readable (size - start));
      else
	logprintf (LOG_VERBOSE, _(", %s remaining"),
		   with_thousand_seps (size - start));
    }
  logputs (LOG_VERBOSE, !authoritative ? _(" (unauthoritative)\n") : "\n");
}

/* Retrieves a file with denoted parameters through opening an FTP
   connection to the server.  It always closes the data connection,
   and closes the control connection in case of error.  */
static uerr_t
getftp (struct url *u, wgint *len, wgint restval, ccon *con)
{
  int csock, dtsock, local_sock, res;
  uerr_t err = RETROK;		/* appease the compiler */
  FILE *fp;
  char *user, *passwd, *respline;
  char *tms, *tmrate;
  int cmd = con->cmd;
  int pasv_mode_open = 0;
  wgint expected_bytes = 0;
  int rest_failed = 0;
  int flags;
  wgint rd_size;

  assert (con != NULL);
  assert (con->target != NULL);

  /* Debug-check of the sanity of the request by making sure that LIST
     and RETR are never both requested (since we can handle only one
     at a time.  */
  assert (!((cmd & DO_LIST) && (cmd & DO_RETR)));
  /* Make sure that at least *something* is requested.  */
  assert ((cmd & (DO_LIST | DO_CWD | DO_RETR | DO_LOGIN)) != 0);

  user = u->user;
  passwd = u->passwd;
  search_netrc (u->host, (const char **)&user, (const char **)&passwd, 1);
  user = user ? user : (opt.ftp_user ? opt.ftp_user : opt.user);
  if (!user) user = "anonymous";
  passwd = passwd ? passwd : (opt.ftp_passwd ? opt.ftp_passwd : opt.passwd);
  if (!passwd) passwd = "-wget@";

  dtsock = -1;
  local_sock = -1;
  con->dltime = 0;

  if (!(cmd & DO_LOGIN))
    csock = con->csock;
  else				/* cmd & DO_LOGIN */
    {
      char type_char;
      char    *host = con->proxy ? con->proxy->host : u->host;
      int      port = con->proxy ? con->proxy->port : u->port;
      char *logname = user;

      if (con->proxy)
	{
	  /* If proxy is in use, log in as username@target-site. */
	  logname = concat_strings (user, "@", u->host, (char *) 0);
	}

      /* Login to the server: */

      /* First: Establish the control connection.  */

      csock = connect_to_host (host, port);
      if (csock == E_HOST)
	return HOSTERR;
      else if (csock < 0)
	return (retryable_socket_connect_error (errno)
		? CONERROR : CONIMPOSSIBLE);

      if (cmd & LEAVE_PENDING)
	con->csock = csock;
      else
	con->csock = -1;

      /* Second: Login with proper USER/PASS sequence.  */
      logprintf (LOG_VERBOSE, _("Logging in as %s ... "), escnonprint (user));
      if (opt.server_response)
	logputs (LOG_ALWAYS, "\n");
      err = ftp_login (csock, logname, passwd);

      if (con->proxy)
	xfree (logname);

      /* FTPRERR, FTPSRVERR, WRITEFAILED, FTPLOGREFUSED, FTPLOGINC */
      switch (err)
	{
	case FTPRERR:
	  logputs (LOG_VERBOSE, "\n");
	  logputs (LOG_NOTQUIET, _("\
Error in server response, closing control connection.\n"));
	  fd_close (csock);
	  con->csock = -1;
	  return err;
	case FTPSRVERR:
	  logputs (LOG_VERBOSE, "\n");
	  logputs (LOG_NOTQUIET, _("Error in server greeting.\n"));
	  fd_close (csock);
	  con->csock = -1;
	  return err;
	case WRITEFAILED:
	  logputs (LOG_VERBOSE, "\n");
	  logputs (LOG_NOTQUIET,
		   _("Write failed, closing control connection.\n"));
	  fd_close (csock);
	  con->csock = -1;
	  return err;
	case FTPLOGREFUSED:
	  logputs (LOG_VERBOSE, "\n");
	  logputs (LOG_NOTQUIET, _("The server refuses login.\n"));
	  fd_close (csock);
	  con->csock = -1;
	  return FTPLOGREFUSED;
	case FTPLOGINC:
	  logputs (LOG_VERBOSE, "\n");
	  logputs (LOG_NOTQUIET, _("Login incorrect.\n"));
	  fd_close (csock);
	  con->csock = -1;
	  return FTPLOGINC;
	case FTPOK:
	  if (!opt.server_response)
	    logputs (LOG_VERBOSE, _("Logged in!\n"));
	  break;
	default:
	  abort ();
	}
      /* Third: Get the system type */
      if (!opt.server_response)
	logprintf (LOG_VERBOSE, "==> SYST ... ");
      err = ftp_syst (csock, &con->rs);
      /* FTPRERR */
      switch (err)
	{
	case FTPRERR:
	  logputs (LOG_VERBOSE, "\n");
	  logputs (LOG_NOTQUIET, _("\
Error in server response, closing control connection.\n"));
	  fd_close (csock);
	  con->csock = -1;
	  return err;
	case FTPSRVERR:
	  logputs (LOG_VERBOSE, "\n");
	  logputs (LOG_NOTQUIET,
		   _("Server error, can't determine system type.\n"));
	  break;
	case FTPOK:
	  /* Everything is OK.  */
	  break;
	default:
	  abort ();
	}
      if (!opt.server_response && err != FTPSRVERR)
	logputs (LOG_VERBOSE, _("done.    "));

      /* Fourth: Find the initial ftp directory */

      if (!opt.server_response)
	logprintf (LOG_VERBOSE, "==> PWD ... ");
      err = ftp_pwd (csock, &con->id);
      /* FTPRERR */
      switch (err)
	{
	case FTPRERR:
	  logputs (LOG_VERBOSE, "\n");
	  logputs (LOG_NOTQUIET, _("\
Error in server response, closing control connection.\n"));
	  fd_close (csock);
	  con->csock = -1;
	  return err;
	case FTPSRVERR :
	  /* PWD unsupported -- assume "/". */
	  xfree_null (con->id);
	  con->id = xstrdup ("/");
	  break;
	case FTPOK:
	  /* Everything is OK.  */
	  break;
	default:
	  abort ();
	}
      /* VMS will report something like "PUB$DEVICE:[INITIAL.FOLDER]".
         Convert it to "/INITIAL/FOLDER" */
      if (con->rs == ST_VMS)
        {
          char *path = strchr (con->id, '[');
	  char *pathend = path ? strchr (path + 1, ']') : NULL;
	  if (!path || !pathend)
	    DEBUGP (("Initial VMS directory not in the form [...]!\n"));
	  else
	    {
	      char *idir = con->id;
	      DEBUGP (("Preprocessing the initial VMS directory\n"));
	      DEBUGP (("  old = '%s'\n", con->id));
	      /* We do the conversion in-place by copying the stuff
		 between [ and ] to the beginning, and changing dots
		 to slashes at the same time.  */
	      *idir++ = '/';
	      for (++path; path < pathend; path++, idir++)
		*idir = *path == '.' ? '/' : *path;
	      *idir = '\0';
	      DEBUGP (("  new = '%s'\n\n", con->id));
	    }
	}
      if (!opt.server_response)
	logputs (LOG_VERBOSE, _("done.\n"));

      /* Fifth: Set the FTP type.  */
      type_char = ftp_process_type (u->params);
      if (!opt.server_response)
	logprintf (LOG_VERBOSE, "==> TYPE %c ... ", type_char);
      err = ftp_type (csock, type_char);
      /* FTPRERR, WRITEFAILED, FTPUNKNOWNTYPE */
      switch (err)
	{
	case FTPRERR:
	  logputs (LOG_VERBOSE, "\n");
	  logputs (LOG_NOTQUIET, _("\
Error in server response, closing control connection.\n"));
	  fd_close (csock);
	  con->csock = -1;
	  return err;
	case WRITEFAILED:
	  logputs (LOG_VERBOSE, "\n");
	  logputs (LOG_NOTQUIET,
		   _("Write failed, closing control connection.\n"));
	  fd_close (csock);
	  con->csock = -1;
	  return err;
	case FTPUNKNOWNTYPE:
	  logputs (LOG_VERBOSE, "\n");
	  logprintf (LOG_NOTQUIET,
		     _("Unknown type `%c', closing control connection.\n"),
		     type_char);
	  fd_close (csock);
	  con->csock = -1;
	  return err;
	case FTPOK:
	  /* Everything is OK.  */
	  break;
	default:
	  abort ();
	}
      if (!opt.server_response)
	logputs (LOG_VERBOSE, _("done.  "));
    } /* do login */

  if (cmd & DO_CWD)
    {
      if (!*u->dir)
	logputs (LOG_VERBOSE, _("==> CWD not needed.\n"));
      else
	{
	  char *target = u->dir;

	  DEBUGP (("changing working directory\n"));

	  /* Change working directory.  To change to a non-absolute
	     Unix directory, we need to prepend initial directory
	     (con->id) to it.  Absolute directories "just work".

	     A relative directory is one that does not begin with '/'
	     and, on non-Unix OS'es, one that doesn't begin with
	     "[a-z]:".

	     This is not done for OS400, which doesn't use
	     "/"-delimited directories, nor does it support directory
	     hierarchies.  "CWD foo" followed by "CWD bar" leaves us
	     in "bar", not in "foo/bar", as would be customary
	     elsewhere.  */

	  if (target[0] != '/'
	      && !(con->rs != ST_UNIX
		   && ISALPHA (target[0])
		   && target[1] == ':')
	      && con->rs != ST_OS400)
	    {
	      int idlen = strlen (con->id);
	      char *ntarget, *p;

	      /* Strip trailing slash(es) from con->id. */
	      while (idlen > 0 && con->id[idlen - 1] == '/')
		--idlen;
	      p = ntarget = (char *)alloca (idlen + 1 + strlen (u->dir) + 1);
	      memcpy (p, con->id, idlen);
	      p += idlen;
	      *p++ = '/';
	      strcpy (p, target);

              DEBUGP (("Prepended initial PWD to relative path:\n"));
              DEBUGP (("   pwd: '%s'\n   old: '%s'\n  new: '%s'\n",
		       con->id, target, ntarget));
	      target = ntarget;
	    }

	  /* If the FTP host runs VMS, we will have to convert the absolute
             directory path in UNIX notation to absolute directory path in
             VMS notation as VMS FTP servers do not like UNIX notation of
             absolute paths.  "VMS notation" is [dir.subdir.subsubdir]. */

	  if (con->rs == ST_VMS)
	    {
	      char *tmpp;
	      char *ntarget = (char *)alloca (strlen (target) + 2);
	      /* We use a converted initial dir, so directories in
                 TARGET will be separated with slashes, something like
                 "/INITIAL/FOLDER/DIR/SUBDIR".  Convert that to
                 "[INITIAL.FOLDER.DIR.SUBDIR]".  */
	      strcpy (ntarget, target);
	      assert (*ntarget == '/');
	      *ntarget = '[';
	      for (tmpp = ntarget + 1; *tmpp; tmpp++)
		if (*tmpp == '/')
		  *tmpp = '.';
	      *tmpp++ = ']';
	      *tmpp = '\0';
              DEBUGP (("Changed file name to VMS syntax:\n"));
              DEBUGP (("  Unix: '%s'\n  VMS: '%s'\n", target, ntarget));
	      target = ntarget;
	    }

	  if (!opt.server_response)
	    logprintf (LOG_VERBOSE, "==> CWD %s ... ", escnonprint (target));
	  err = ftp_cwd (csock, target);
	  /* FTPRERR, WRITEFAILED, FTPNSFOD */
	  switch (err)
	    {
	    case FTPRERR:
	      logputs (LOG_VERBOSE, "\n");
	      logputs (LOG_NOTQUIET, _("\
Error in server response, closing control connection.\n"));
	      fd_close (csock);
	      con->csock = -1;
	      return err;
	    case WRITEFAILED:
	      logputs (LOG_VERBOSE, "\n");
	      logputs (LOG_NOTQUIET,
		       _("Write failed, closing control connection.\n"));
	      fd_close (csock);
	      con->csock = -1;
	      return err;
	    case FTPNSFOD:
	      logputs (LOG_VERBOSE, "\n");
	      logprintf (LOG_NOTQUIET, _("No such directory `%s'.\n\n"),
			 escnonprint (u->dir));
	      fd_close (csock);
	      con->csock = -1;
	      return err;
	    case FTPOK:
	      break;
	    default:
	      abort ();
	    }
	  if (!opt.server_response)
	    logputs (LOG_VERBOSE, _("done.\n"));
	}
    }
  else /* do not CWD */
    logputs (LOG_VERBOSE, _("==> CWD not required.\n"));

  if ((cmd & DO_RETR) && restval && *len == 0)
    {
      if (opt.verbose)
	{
          if (!opt.server_response)
	    logprintf (LOG_VERBOSE, "==> SIZE %s ... ", escnonprint (u->file));
	}

      err = ftp_size (csock, u->file, len);
      /* FTPRERR */
      switch (err)
	{
	case FTPRERR:
	case FTPSRVERR :
	  logputs (LOG_VERBOSE, "\n");
	  logputs (LOG_NOTQUIET, _("\
Error in server response, closing control connection.\n"));
	  fd_close (csock);
	  con->csock = -1;
	  return err;
	case FTPOK:
	  /* Everything is OK.  */
	  break;
	default:
	  abort ();
	}
	if (!opt.server_response)
	  logputs (LOG_VERBOSE, _("done.\n"));
    }

  /* If anything is to be retrieved, PORT (or PASV) must be sent.  */
  if (cmd & (DO_LIST | DO_RETR))
    {
      if (opt.ftp_pasv > 0)
	{
  	  ip_address passive_addr;
  	  int        passive_port;
	  err = ftp_do_pasv (csock, &passive_addr, &passive_port);
	  /* FTPRERR, WRITEFAILED, FTPNOPASV, FTPINVPASV */
	  switch (err)
	    {
	    case FTPRERR:
	      logputs (LOG_VERBOSE, "\n");
	      logputs (LOG_NOTQUIET, _("\
Error in server response, closing control connection.\n"));
	      fd_close (csock);
	      con->csock = -1;
	      return err;
	    case WRITEFAILED:
	      logputs (LOG_VERBOSE, "\n");
	      logputs (LOG_NOTQUIET,
		       _("Write failed, closing control connection.\n"));
	      fd_close (csock);
	      con->csock = -1;
	      return err;
	    case FTPNOPASV:
	      logputs (LOG_VERBOSE, "\n");
	      logputs (LOG_NOTQUIET, _("Cannot initiate PASV transfer.\n"));
	      break;
	    case FTPINVPASV:
	      logputs (LOG_VERBOSE, "\n");
	      logputs (LOG_NOTQUIET, _("Cannot parse PASV response.\n"));
	      break;
	    case FTPOK:
	      break;
	    default:
	      abort ();
	    }	/* switch (err) */
	  if (err==FTPOK)
	    {
	      DEBUGP (("trying to connect to %s port %d\n",
		      pretty_print_address (&passive_addr),
		      passive_port));
	      dtsock = connect_to_ip (&passive_addr, passive_port, NULL);
	      if (dtsock < 0)
		{
		  int save_errno = errno;
		  fd_close (csock);
		  con->csock = -1;
		  logprintf (LOG_VERBOSE, _("couldn't connect to %s port %d: %s\n"),
			     pretty_print_address (&passive_addr), passive_port,
			     strerror (save_errno));
		  return (retryable_socket_connect_error (save_errno)
			  ? CONERROR : CONIMPOSSIBLE);
		}

	      pasv_mode_open = 1;  /* Flag to avoid accept port */
	      if (!opt.server_response)
		logputs (LOG_VERBOSE, _("done.    "));
	    } /* err==FTP_OK */
	}

      if (!pasv_mode_open)   /* Try to use a port command if PASV failed */
	{
	  err = ftp_do_port (csock, &local_sock);
	  /* FTPRERR, WRITEFAILED, bindport (FTPSYSERR), HOSTERR,
	     FTPPORTERR */
	  switch (err)
	    {
	    case FTPRERR:
	      logputs (LOG_VERBOSE, "\n");
	      logputs (LOG_NOTQUIET, _("\
Error in server response, closing control connection.\n"));
	      fd_close (csock);
	      con->csock = -1;
	      fd_close (dtsock);
	      fd_close (local_sock);
	      return err;
	    case WRITEFAILED:
	      logputs (LOG_VERBOSE, "\n");
	      logputs (LOG_NOTQUIET,
		       _("Write failed, closing control connection.\n"));
	      fd_close (csock);
	      con->csock = -1;
	      fd_close (dtsock);
	      fd_close (local_sock);
	      return err;
	    case CONSOCKERR:
	      logputs (LOG_VERBOSE, "\n");
	      logprintf (LOG_NOTQUIET, "socket: %s\n", strerror (errno));
	      fd_close (csock);
	      con->csock = -1;
	      fd_close (dtsock);
	      fd_close (local_sock);
	      return err;
	    case FTPSYSERR:
	      logputs (LOG_VERBOSE, "\n");
	      logprintf (LOG_NOTQUIET, _("Bind error (%s).\n"),
			 strerror (errno));
	      fd_close (dtsock);
	      return err;
	    case FTPPORTERR:
	      logputs (LOG_VERBOSE, "\n");
	      logputs (LOG_NOTQUIET, _("Invalid PORT.\n"));
	      fd_close (csock);
	      con->csock = -1;
	      fd_close (dtsock);
	      fd_close (local_sock);
	      return err;
	    case FTPOK:
	      break;
	    default:
	      abort ();
	    } /* port switch */
	  if (!opt.server_response)
	    logputs (LOG_VERBOSE, _("done.    "));
	} /* dtsock == -1 */
    } /* cmd & (DO_LIST | DO_RETR) */

  /* Restart if needed.  */
  if (restval && (cmd & DO_RETR))
    {
      if (!opt.server_response)
	logprintf (LOG_VERBOSE, "==> REST %s ... ",
		   number_to_static_string (restval));
      err = ftp_rest (csock, restval);

      /* FTPRERR, WRITEFAILED, FTPRESTFAIL */
      switch (err)
	{
	case FTPRERR:
	  logputs (LOG_VERBOSE, "\n");
	  logputs (LOG_NOTQUIET, _("\
Error in server response, closing control connection.\n"));
	  fd_close (csock);
	  con->csock = -1;
	  fd_close (dtsock);
	  fd_close (local_sock);
	  return err;
	case WRITEFAILED:
	  logputs (LOG_VERBOSE, "\n");
	  logputs (LOG_NOTQUIET,
		   _("Write failed, closing control connection.\n"));
	  fd_close (csock);
	  con->csock = -1;
	  fd_close (dtsock);
	  fd_close (local_sock);
	  return err;
	case FTPRESTFAIL:
	  logputs (LOG_VERBOSE, _("\nREST failed, starting from scratch.\n"));
	  rest_failed = 1;
	  break;
	case FTPOK:
	  break;
	default:
	  abort ();
	}
      if (err != FTPRESTFAIL && !opt.server_response)
	logputs (LOG_VERBOSE, _("done.    "));
    } /* restval && cmd & DO_RETR */

  if (cmd & DO_RETR)
    {
      /* If we're in spider mode, don't really retrieve anything.  The
	 fact that we got to this point should be proof enough that
	 the file exists, vaguely akin to HTTP's concept of a "HEAD"
	 request.  */
      if (opt.spider)
	{
	  fd_close (csock);
	  con->csock = -1;
	  fd_close (dtsock);
	  fd_close (local_sock);
	  return RETRFINISHED;
	}

      if (opt.verbose)
	{
	  if (!opt.server_response)
	    {
	      if (restval)
		logputs (LOG_VERBOSE, "\n");
	      logprintf (LOG_VERBOSE, "==> RETR %s ... ", escnonprint (u->file));
	    }
	}

      err = ftp_retr (csock, u->file);
      /* FTPRERR, WRITEFAILED, FTPNSFOD */
      switch (err)
	{
	case FTPRERR:
	  logputs (LOG_VERBOSE, "\n");
	  logputs (LOG_NOTQUIET, _("\
Error in server response, closing control connection.\n"));
	  fd_close (csock);
	  con->csock = -1;
	  fd_close (dtsock);
	  fd_close (local_sock);
	  return err;
	case WRITEFAILED:
	  logputs (LOG_VERBOSE, "\n");
	  logputs (LOG_NOTQUIET,
		   _("Write failed, closing control connection.\n"));
	  fd_close (csock);
	  con->csock = -1;
	  fd_close (dtsock);
	  fd_close (local_sock);
	  return err;
	case FTPNSFOD:
	  logputs (LOG_VERBOSE, "\n");
	  logprintf (LOG_NOTQUIET, _("No such file `%s'.\n\n"),
		     escnonprint (u->file));
	  fd_close (dtsock);
	  fd_close (local_sock);
	  return err;
	case FTPOK:
	  break;
	default:
	  abort ();
	}

      if (!opt.server_response)
	logputs (LOG_VERBOSE, _("done.\n"));
      expected_bytes = ftp_expected_bytes (ftp_last_respline);
    } /* do retrieve */

  if (cmd & DO_LIST)
    {
      if (!opt.server_response)
	logputs (LOG_VERBOSE, "==> LIST ... ");
      /* As Maciej W. Rozycki (macro@ds2.pg.gda.pl) says, `LIST'
	 without arguments is better than `LIST .'; confirmed by
	 RFC959.  */
      err = ftp_list (csock, NULL);
      /* FTPRERR, WRITEFAILED */
      switch (err)
	{
	case FTPRERR:
	  logputs (LOG_VERBOSE, "\n");
	  logputs (LOG_NOTQUIET, _("\
Error in server response, closing control connection.\n"));
	  fd_close (csock);
	  con->csock = -1;
	  fd_close (dtsock);
	  fd_close (local_sock);
	  return err;
	case WRITEFAILED:
	  logputs (LOG_VERBOSE, "\n");
	  logputs (LOG_NOTQUIET,
		   _("Write failed, closing control connection.\n"));
	  fd_close (csock);
	  con->csock = -1;
	  fd_close (dtsock);
	  fd_close (local_sock);
	  return err;
	case FTPNSFOD:
	  logputs (LOG_VERBOSE, "\n");
	  logprintf (LOG_NOTQUIET, _("No such file or directory `%s'.\n\n"),
		     ".");
	  fd_close (dtsock);
	  fd_close (local_sock);
	  return err;
	case FTPOK:
	  break;
	default:
	  abort ();
	}
      if (!opt.server_response)
	logputs (LOG_VERBOSE, _("done.\n"));
      expected_bytes = ftp_expected_bytes (ftp_last_respline);
    } /* cmd & DO_LIST */

  if (!(cmd & (DO_LIST | DO_RETR)) || (opt.spider && !(cmd & DO_LIST)))
    return RETRFINISHED;

  /* Some FTP servers return the total length of file after REST
     command, others just return the remaining size. */
  if (*len && restval && expected_bytes
      && (expected_bytes == *len - restval))
    {
      DEBUGP (("Lying FTP server found, adjusting.\n"));
      expected_bytes = *len;
    }

  /* If no transmission was required, then everything is OK.  */
  if (!pasv_mode_open)  /* we are not using pasive mode so we need
			      to accept */
    {
      /* Wait for the server to connect to the address we're waiting
	 at.  */
      dtsock = accept_connection (local_sock);
      if (dtsock < 0)
	{
	  logprintf (LOG_NOTQUIET, "accept: %s\n", strerror (errno));
	  return err;
	}
    }

  /* Open the file -- if output_stream is set, use it instead.  */
  if (!output_stream || con->cmd & DO_LIST)
    {
      mkalldirs (con->target);
      if (opt.backups)
	rotate_backups (con->target);

      if (restval)
	fp = fopen (con->target, "ab");
      else if (opt.noclobber || opt.always_rest || opt.timestamping || opt.dirstruct
	       || opt.output_document)
	fp = fopen (con->target, "wb");
      else
	{
	  fp = fopen_excl (con->target, 1);
	  if (!fp && errno == EEXIST)
	    {
	      /* We cannot just invent a new name and use it (which is
		 what functions like unique_create typically do)
		 because we told the user we'd use this name.
		 Instead, return and retry the download.  */
	      logprintf (LOG_NOTQUIET, _("%s has sprung into existence.\n"),
			 con->target);
	      fd_close (csock);
	      con->csock = -1;
	      fd_close (dtsock);
	      fd_close (local_sock);
	      return FOPEN_EXCL_ERR;
	    }
	}
      if (!fp)
	{
	  logprintf (LOG_NOTQUIET, "%s: %s\n", con->target, strerror (errno));
	  fd_close (csock);
	  con->csock = -1;
	  fd_close (dtsock);
	  fd_close (local_sock);
	  return FOPENERR;
	}
    }
  else
    fp = output_stream;

  if (*len)
    {
      print_length (*len, restval, 1);
      expected_bytes = *len;	/* for get_contents/show_progress */
    }
  else if (expected_bytes)
    print_length (expected_bytes, restval, 0);

  /* Get the contents of the document.  */
  flags = 0;
  if (restval && rest_failed)
    flags |= rb_skip_startpos;
  *len = restval;
  rd_size = 0;
  res = fd_read_body (dtsock, fp,
		      expected_bytes ? expected_bytes - restval : 0,
		      restval, &rd_size, len, &con->dltime, flags);

  tms = time_str (NULL);
  tmrate = retr_rate (rd_size, con->dltime, 0);
  /* Close data connection socket.  */
  fd_close (dtsock);
  fd_close (local_sock);
  /* Close the local file.  */
  {
    /* Close or flush the file.  We have to be careful to check for
       error here.  Checking the result of fwrite() is not enough --
       errors could go unnoticed!  */
    int flush_res;
    if (!output_stream || con->cmd & DO_LIST)
      flush_res = fclose (fp);
    else
      flush_res = fflush (fp);
    if (flush_res == EOF)
      res = -2;
  }

  /* If get_contents couldn't write to fp, bail out.  */
  if (res == -2)
    {
      logprintf (LOG_NOTQUIET, _("%s: %s, closing control connection.\n"),
		 con->target, strerror (errno));
      fd_close (csock);
      con->csock = -1;
      return FWRITEERR;
    }
  else if (res == -1)
    {
      logprintf (LOG_NOTQUIET, _("%s (%s) - Data connection: %s; "),
		 tms, tmrate, strerror (errno));
      if (opt.server_response)
	logputs (LOG_ALWAYS, "\n");
    }

  /* Get the server to tell us if everything is retrieved.  */
  err = ftp_response (csock, &respline);
  if (err != FTPOK)
    {
      /* The control connection is decidedly closed.  Print the time
	 only if it hasn't already been printed.  */
      if (res != -1)
	logprintf (LOG_NOTQUIET, "%s (%s) - ", tms, tmrate);
      logputs (LOG_NOTQUIET, _("Control connection closed.\n"));
      /* If there is an error on the control connection, close it, but
	 return FTPRETRINT, since there is a possibility that the
	 whole file was retrieved nevertheless (but that is for
	 ftp_loop_internal to decide).  */
      fd_close (csock);
      con->csock = -1;
      return FTPRETRINT;
    } /* err != FTPOK */
  /* If retrieval failed for any reason, return FTPRETRINT, but do not
     close socket, since the control connection is still alive.  If
     there is something wrong with the control connection, it will
     become apparent later.  */
  if (*respline != '2')
    {
      xfree (respline);
      if (res != -1)
	logprintf (LOG_NOTQUIET, "%s (%s) - ", tms, tmrate);
      logputs (LOG_NOTQUIET, _("Data transfer aborted.\n"));
      return FTPRETRINT;
    }
  xfree (respline);

  if (res == -1)
    {
      /* What now?  The data connection was erroneous, whereas the
	 response says everything is OK.  We shall play it safe.  */
      return FTPRETRINT;
    }

  if (!(cmd & LEAVE_PENDING))
    {
      /* Closing the socket is faster than sending 'QUIT' and the
	 effect is the same.  */
      fd_close (csock);
      con->csock = -1;
    }
  /* If it was a listing, and opt.server_response is true,
     print it out.  */
  if (opt.server_response && (con->cmd & DO_LIST))
    {
      mkalldirs (con->target);
      fp = fopen (con->target, "r");
      if (!fp)
	logprintf (LOG_ALWAYS, "%s: %s\n", con->target, strerror (errno));
      else
	{
	  char *line;
	  /* The lines are being read with read_whole_line because of
	     no-buffering on opt.lfile.  */
	  while ((line = read_whole_line (fp)) != NULL)
	    {
	      char *p = strchr (line, '\0');
	      while (p > line && (p[-1] == '\n' || p[-1] == '\r'))
		*--p = '\0';
	      logprintf (LOG_ALWAYS, "%s\n", escnonprint (line));
	      xfree (line);
	    }
	  fclose (fp);
	}
    } /* con->cmd & DO_LIST && server_response */

  return RETRFINISHED;
}

/* A one-file FTP loop.  This is the part where FTP retrieval is
   retried, and retried, and retried, and...

   This loop either gets commands from con, or (if ON_YOUR_OWN is
   set), makes them up to retrieve the file given by the URL.  */
static uerr_t
ftp_loop_internal (struct url *u, struct fileinfo *f, ccon *con)
{
  int count, orig_lp;
  wgint restval, len = 0;
  char *tms, *locf;
  char *tmrate = NULL;
  uerr_t err;
  struct_stat st;

  if (!con->target)
    con->target = url_file_name (u);

  if (opt.noclobber && file_exists_p (con->target))
    {
      logprintf (LOG_VERBOSE,
		 _("File `%s' already there; not retrieving.\n"), con->target);
      /* If the file is there, we suppose it's retrieved OK.  */
      return RETROK;
    }

  /* Remove it if it's a link.  */
  remove_link (con->target);
  if (!opt.output_document)
    locf = con->target;
  else
    locf = opt.output_document;

  count = 0;

  if (con->st & ON_YOUR_OWN)
    con->st = ON_YOUR_OWN;

  orig_lp = con->cmd & LEAVE_PENDING ? 1 : 0;

  /* THE loop.  */
  do
    {
      /* Increment the pass counter.  */
      ++count;
      sleep_between_retrievals (count);
      if (con->st & ON_YOUR_OWN)
	{
	  con->cmd = 0;
	  con->cmd |= (DO_RETR | LEAVE_PENDING);
	  if (con->csock != -1)
	    con->cmd &= ~ (DO_LOGIN | DO_CWD);
	  else
	    con->cmd |= (DO_LOGIN | DO_CWD);
	}
      else /* not on your own */
	{
	  if (con->csock != -1)
	    con->cmd &= ~DO_LOGIN;
	  else
	    con->cmd |= DO_LOGIN;
	  if (con->st & DONE_CWD)
	    con->cmd &= ~DO_CWD;
	  else
	    con->cmd |= DO_CWD;
	}

      /* Decide whether or not to restart.  */
      if (opt.always_rest
	  && stat (locf, &st) == 0
	  && S_ISREG (st.st_mode))
	/* When -c is used, continue from on-disk size.  (Can't use
	   hstat.len even if count>1 because we don't want a failed
	   first attempt to clobber existing data.)  */
	restval = st.st_size;
      else if (count > 1)
	restval = len;		/* start where the previous run left off */
      else
	restval = 0;

      /* Get the current time string.  */
      tms = time_str (NULL);
      /* Print fetch message, if opt.verbose.  */
      if (opt.verbose)
	{
	  char *hurl = url_string (u, 1);
	  char tmp[256];
	  strcpy (tmp, "        ");
	  if (count > 1)
	    sprintf (tmp, _("(try:%2d)"), count);
	  logprintf (LOG_VERBOSE, "--%s--  %s\n  %s => `%s'\n",
		     tms, hurl, tmp, locf);
#ifdef WINDOWS
	  ws_changetitle (hurl);
#endif
	  xfree (hurl);
	}
      /* Send getftp the proper length, if fileinfo was provided.  */
      if (f)
	len = f->size;
      else
	len = 0;
      err = getftp (u, &len, restval, con);

      if (con->csock != -1)
	con->st &= ~DONE_CWD;
      else
	con->st |= DONE_CWD;

      switch (err)
	{
	case HOSTERR: case CONIMPOSSIBLE: case FWRITEERR: case FOPENERR:
	case FTPNSFOD: case FTPLOGINC: case FTPNOPASV: case CONTNOTSUPPORTED:
	  /* Fatal errors, give up.  */
	  return err;
	case CONSOCKERR: case CONERROR: case FTPSRVERR: case FTPRERR:
	case WRITEFAILED: case FTPUNKNOWNTYPE: case FTPSYSERR:
	case FTPPORTERR: case FTPLOGREFUSED: case FTPINVPASV:
	case FOPEN_EXCL_ERR:
	  printwhat (count, opt.ntry);
	  /* non-fatal errors */
	  if (err == FOPEN_EXCL_ERR)
	    {
	      /* Re-determine the file name. */
	      xfree_null (con->target);
	      con->target = url_file_name (u);
	      locf = con->target;
	    }
	  continue;
	case FTPRETRINT:
	  /* If the control connection was closed, the retrieval
	     will be considered OK if f->size == len.  */
	  if (!f || len != f->size)
	    {
	      printwhat (count, opt.ntry);
	      continue;
	    }
	  break;
	case RETRFINISHED:
	  /* Great!  */
	  break;
	default:
	  /* Not as great.  */
	  abort ();
	}
      tms = time_str (NULL);
      if (!opt.spider)
        tmrate = retr_rate (len - restval, con->dltime, 0);

      /* If we get out of the switch above without continue'ing, we've
	 successfully downloaded a file.  Remember this fact. */
      downloaded_file (FILE_DOWNLOADED_NORMALLY, locf);

      if (con->st & ON_YOUR_OWN)
	{
	  fd_close (con->csock);
	  con->csock = -1;
	}
      if (!opt.spider)
        logprintf (LOG_VERBOSE, _("%s (%s) - `%s' saved [%s]\n\n"),
		   tms, tmrate, locf, number_to_static_string (len));
      if (!opt.verbose && !opt.quiet)
	{
	  /* Need to hide the password from the URL.  The `if' is here
             so that we don't do the needless allocation every
             time. */
	  char *hurl = url_string (u, 1);
	  logprintf (LOG_NONVERBOSE, "%s URL: %s [%s] -> \"%s\" [%d]\n",
		     tms, hurl, number_to_static_string (len), locf, count);
	  xfree (hurl);
	}

      if ((con->cmd & DO_LIST))
	/* This is a directory listing file. */
	{
	  if (!opt.remove_listing)
	    /* --dont-remove-listing was specified, so do count this towards the
	       number of bytes and files downloaded. */
	    {
	      total_downloaded_bytes += len;
	      opt.numurls++;
	    }

	  /* Deletion of listing files is not controlled by --delete-after, but
	     by the more specific option --dont-remove-listing, and the code
	     to do this deletion is in another function. */
	}
      else if (!opt.spider)
	/* This is not a directory listing file. */
	{
	  /* Unlike directory listing files, don't pretend normal files weren't
	     downloaded if they're going to be deleted.  People seeding proxies,
	     for instance, may want to know how many bytes and files they've
	     downloaded through it. */
	  total_downloaded_bytes += len;
	  opt.numurls++;

	  if (opt.delete_after)
	    {
	      DEBUGP (("\
Removing file due to --delete-after in ftp_loop_internal():\n"));
	      logprintf (LOG_VERBOSE, _("Removing %s.\n"), locf);
	      if (unlink (locf))
		logprintf (LOG_NOTQUIET, "unlink: %s\n", strerror (errno));
	    }
	}

      /* Restore the original leave-pendingness.  */
      if (orig_lp)
	con->cmd |= LEAVE_PENDING;
      else
	con->cmd &= ~LEAVE_PENDING;
      return RETROK;
    } while (!opt.ntry || (count < opt.ntry));

  if (con->csock != -1 && (con->st & ON_YOUR_OWN))
    {
      fd_close (con->csock);
      con->csock = -1;
    }
  return TRYLIMEXC;
}

/* Return the directory listing in a reusable format.  The directory
   is specifed in u->dir.  */
uerr_t
ftp_get_listing (struct url *u, ccon *con, struct fileinfo **f)
{
  uerr_t err;
  char *uf;			/* url file name */
  char *lf;			/* list file name */
  char *old_target = con->target;

  con->st &= ~ON_YOUR_OWN;
  con->cmd |= (DO_LIST | LEAVE_PENDING);
  con->cmd &= ~DO_RETR;

  /* Find the listing file name.  We do it by taking the file name of
     the URL and replacing the last component with the listing file
     name.  */
  uf = url_file_name (u);
  lf = file_merge (uf, LIST_FILENAME);
  xfree (uf);
  DEBUGP ((_("Using `%s' as listing tmp file.\n"), lf));

  con->target = lf;
  err = ftp_loop_internal (u, NULL, con);
  con->target = old_target;

  if (err == RETROK)
    *f = ftp_parse_ls (lf, con->rs);
  else
    *f = NULL;
  if (opt.remove_listing)
    {
      if (unlink (lf))
	logprintf (LOG_NOTQUIET, "unlink: %s\n", strerror (errno));
      else
	logprintf (LOG_VERBOSE, _("Removed `%s'.\n"), lf);
    }
  xfree (lf);
  con->cmd &= ~DO_LIST;
  return err;
}

static uerr_t ftp_retrieve_dirs PARAMS ((struct url *, struct fileinfo *,
					 ccon *));
static uerr_t ftp_retrieve_glob PARAMS ((struct url *, ccon *, int));
static struct fileinfo *delelement PARAMS ((struct fileinfo *,
					    struct fileinfo **));
static void freefileinfo PARAMS ((struct fileinfo *f));

/* Retrieve a list of files given in struct fileinfo linked list.  If
   a file is a symbolic link, do not retrieve it, but rather try to
   set up a similar link on the local disk, if the symlinks are
   supported.

   If opt.recursive is set, after all files have been retrieved,
   ftp_retrieve_dirs will be called to retrieve the directories.  */
static uerr_t
ftp_retrieve_list (struct url *u, struct fileinfo *f, ccon *con)
{
  static int depth = 0;
  uerr_t err;
  struct fileinfo *orig;
  wgint local_size;
  time_t tml;
  int dlthis;

  /* Increase the depth.  */
  ++depth;
  if (opt.reclevel != INFINITE_RECURSION && depth > opt.reclevel)
    {
      DEBUGP ((_("Recursion depth %d exceeded max. depth %d.\n"),
	       depth, opt.reclevel));
      --depth;
      return RECLEVELEXC;
    }

  assert (f != NULL);
  orig = f;

  con->st &= ~ON_YOUR_OWN;
  if (!(con->st & DONE_CWD))
    con->cmd |= DO_CWD;
  else
    con->cmd &= ~DO_CWD;
  con->cmd |= (DO_RETR | LEAVE_PENDING);

  if (con->csock < 0)
    con->cmd |= DO_LOGIN;
  else
    con->cmd &= ~DO_LOGIN;

  err = RETROK;			/* in case it's not used */

  while (f)
    {
      char *old_target, *ofile;

      if (opt.quota && total_downloaded_bytes > opt.quota)
	{
	  --depth;
	  return QUOTEXC;
	}
      old_target = con->target;

      ofile = xstrdup (u->file);
      url_set_file (u, f->name);

      con->target = url_file_name (u);
      err = RETROK;

      dlthis = 1;
      if (opt.timestamping && f->type == FT_PLAINFILE)
        {
	  struct_stat st;
	  /* If conversion of HTML files retrieved via FTP is ever implemented,
	     we'll need to stat() <file>.orig here when -K has been specified.
	     I'm not implementing it now since files on an FTP server are much
	     more likely than files on an HTTP server to legitimately have a
	     .orig suffix. */
	  if (!stat (con->target, &st))
	    {
              int eq_size;
              int cor_val;
	      /* Else, get it from the file.  */
	      local_size = st.st_size;
	      tml = st.st_mtime;
#ifdef WINDOWS
	      /* Modification time granularity is 2 seconds for Windows, so
		 increase local time by 1 second for later comparison. */
	      tml++;
#endif
              /* Compare file sizes only for servers that tell us correct
                 values. Assumme sizes being equal for servers that lie
                 about file size.  */
              cor_val = (con->rs == ST_UNIX || con->rs == ST_WINNT);
              eq_size = cor_val ? (local_size == f->size) : 1 ;
	      if (f->tstamp <= tml && eq_size)
		{
		  /* Remote file is older, file sizes can be compared and
                     are both equal. */
                  logprintf (LOG_VERBOSE, _("\
Remote file no newer than local file `%s' -- not retrieving.\n"), con->target);
		  dlthis = 0;
		}
	      else if (eq_size)
                {
                  /* Remote file is newer or sizes cannot be matched */
                  logprintf (LOG_VERBOSE, _("\
Remote file is newer than local file `%s' -- retrieving.\n\n"),
                             con->target);
                }
              else
                {
                  /* Sizes do not match */
                  logprintf (LOG_VERBOSE, _("\
The sizes do not match (local %s) -- retrieving.\n\n"),
			     number_to_static_string (local_size));
                }
            }
	}	/* opt.timestamping && f->type == FT_PLAINFILE */
      switch (f->type)
	{
	case FT_SYMLINK:
	  /* If opt.retr_symlinks is defined, we treat symlinks as
	     if they were normal files.  There is currently no way
	     to distinguish whether they might be directories, and
	     follow them.  */
	  if (!opt.retr_symlinks)
	    {
#ifdef HAVE_SYMLINK
	      if (!f->linkto)
		logputs (LOG_NOTQUIET,
			 _("Invalid name of the symlink, skipping.\n"));
	      else
		{
                  struct_stat st;
		  /* Check whether we already have the correct
                     symbolic link.  */
                  int rc = lstat (con->target, &st);
                  if (rc == 0)
		    {
		      size_t len = strlen (f->linkto) + 1;
		      if (S_ISLNK (st.st_mode))
			{
			  char *link_target = (char *)alloca (len);
			  size_t n = readlink (con->target, link_target, len);
			  if ((n == len - 1)
			      && (memcmp (link_target, f->linkto, n) == 0))
			    {
			      logprintf (LOG_VERBOSE, _("\
Already have correct symlink %s -> %s\n\n"),
					 con->target, escnonprint (f->linkto));
                              dlthis = 0;
			      break;
			    }
			}
		    }
		  logprintf (LOG_VERBOSE, _("Creating symlink %s -> %s\n"),
			     con->target, escnonprint (f->linkto));
		  /* Unlink before creating symlink!  */
		  unlink (con->target);
		  if (symlink (f->linkto, con->target) == -1)
		    logprintf (LOG_NOTQUIET, "symlink: %s\n", strerror (errno));
		  logputs (LOG_VERBOSE, "\n");
		} /* have f->linkto */
#else  /* not HAVE_SYMLINK */
	      logprintf (LOG_NOTQUIET,
			 _("Symlinks not supported, skipping symlink `%s'.\n"),
			 con->target);
#endif /* not HAVE_SYMLINK */
	    }
	  else                /* opt.retr_symlinks */
	    {
	      if (dlthis)
		err = ftp_loop_internal (u, f, con);
	    } /* opt.retr_symlinks */
	  break;
	case FT_DIRECTORY:
	  if (!opt.recursive)
	    logprintf (LOG_NOTQUIET, _("Skipping directory `%s'.\n"),
		       escnonprint (f->name));
	  break;
	case FT_PLAINFILE:
	  /* Call the retrieve loop.  */
	  if (dlthis)
	    err = ftp_loop_internal (u, f, con);
	  break;
	case FT_UNKNOWN:
	  logprintf (LOG_NOTQUIET, _("%s: unknown/unsupported file type.\n"),
		     escnonprint (f->name));
	  break;
	}	/* switch */

      /* Set the time-stamp information to the local file.  Symlinks
	 are not to be stamped because it sets the stamp on the
	 original.  :( */
      if (!(f->type == FT_SYMLINK && !opt.retr_symlinks)
	  && f->tstamp != -1
          && dlthis
	  && file_exists_p (con->target))
	{
	  /* #### This code repeats in http.c and ftp.c.  Move it to a
             function!  */
	  const char *fl = NULL;
	  if (opt.output_document)
	    {
	      if (output_stream_regular)
		fl = opt.output_document;
	    }
	  else
	    fl = con->target;
	  if (fl)
	    touch (fl, f->tstamp);
	}
      else if (f->tstamp == -1)
	logprintf (LOG_NOTQUIET, _("%s: corrupt time-stamp.\n"), con->target);

      if (f->perms && f->type == FT_PLAINFILE && dlthis)
        {
	  if (opt.preserve_perm)
	    chmod (con->target, f->perms);
        }
      else
	DEBUGP (("Unrecognized permissions for %s.\n", con->target));

      xfree (con->target);
      con->target = old_target;

      url_set_file (u, ofile);
      xfree (ofile);

      /* Break on fatals.  */
      if (err == QUOTEXC || err == HOSTERR || err == FWRITEERR)
	break;
      con->cmd &= ~ (DO_CWD | DO_LOGIN);
      f = f->next;
    }

  /* We do not want to call ftp_retrieve_dirs here */
  if (opt.recursive &&
      !(opt.reclevel != INFINITE_RECURSION && depth >= opt.reclevel))
    err = ftp_retrieve_dirs (u, orig, con);
  else if (opt.recursive)
    DEBUGP ((_("Will not retrieve dirs since depth is %d (max %d).\n"),
	     depth, opt.reclevel));
  --depth;
  return err;
}

/* Retrieve the directories given in a file list.  This function works
   by simply going through the linked list and calling
   ftp_retrieve_glob on each directory entry.  The function knows
   about excluded directories.  */
static uerr_t
ftp_retrieve_dirs (struct url *u, struct fileinfo *f, ccon *con)
{
  char *container = NULL;
  int container_size = 0;

  for (; f; f = f->next)
    {
      int size;
      char *odir, *newdir;

      if (opt.quota && total_downloaded_bytes > opt.quota)
	break;
      if (f->type != FT_DIRECTORY)
	continue;

      /* Allocate u->dir off stack, but reallocate only if a larger
         string is needed.  It's a pity there's no "realloca" for an
         item on the bottom of the stack.  */
      size = strlen (u->dir) + 1 + strlen (f->name) + 1;
      if (size > container_size)
	container = (char *)alloca (size);
      newdir = container;

      odir = u->dir;
      if (*odir == '\0'
	  || (*odir == '/' && *(odir + 1) == '\0'))
	/* If ODIR is empty or just "/", simply append f->name to
	   ODIR.  (In the former case, to preserve u->dir being
	   relative; in the latter case, to avoid double slash.)  */
	sprintf (newdir, "%s%s", odir, f->name);
      else
	/* Else, use a separator. */
	sprintf (newdir, "%s/%s", odir, f->name);

      DEBUGP (("Composing new CWD relative to the initial directory.\n"));
      DEBUGP (("  odir = '%s'\n  f->name = '%s'\n  newdir = '%s'\n\n",
	       odir, f->name, newdir));
      if (!accdir (newdir, ALLABS))
	{
	  logprintf (LOG_VERBOSE, _("\
Not descending to `%s' as it is excluded/not-included.\n"),
		     escnonprint (newdir));
	  continue;
	}

      con->st &= ~DONE_CWD;

      odir = xstrdup (u->dir);	/* because url_set_dir will free
				   u->dir. */
      url_set_dir (u, newdir);
      ftp_retrieve_glob (u, con, GLOB_GETALL);
      url_set_dir (u, odir);
      xfree (odir);

      /* Set the time-stamp?  */
    }

  if (opt.quota && total_downloaded_bytes > opt.quota)
    return QUOTEXC;
  else
    return RETROK;
}

/* Return non-zero if S has a leading '/'  or contains '../' */
static int
has_insecure_name_p (const char *s)
{
  if (*s == '/')
    return 1;

  if (strstr (s, "../") != 0)
    return 1;

  return 0;
}

/* A near-top-level function to retrieve the files in a directory.
   The function calls ftp_get_listing, to get a linked list of files.
   Then it weeds out the file names that do not match the pattern.
   ftp_retrieve_list is called with this updated list as an argument.

   If the argument ACTION is GLOB_GETONE, just download the file (but
   first get the listing, so that the time-stamp is heeded); if it's
   GLOB_GLOBALL, use globbing; if it's GLOB_GETALL, download the whole
   directory.  */
static uerr_t
ftp_retrieve_glob (struct url *u, ccon *con, int action)
{
  struct fileinfo *f, *start;
  uerr_t res;

  con->cmd |= LEAVE_PENDING;

  res = ftp_get_listing (u, con, &start);
  if (res != RETROK)
    return res;
  /* First: weed out that do not conform the global rules given in
     opt.accepts and opt.rejects.  */
  if (opt.accepts || opt.rejects)
    {
      f = start;
      while (f)
	{
	  if (f->type != FT_DIRECTORY && !acceptable (f->name))
	    {
	      logprintf (LOG_VERBOSE, _("Rejecting `%s'.\n"),
			 escnonprint (f->name));
	      f = delelement (f, &start);
	    }
	  else
	    f = f->next;
	}
    }
  /* Remove all files with possible harmful names */
  f = start;
  while (f)
    {
      if (has_insecure_name_p (f->name))
	{
	  logprintf (LOG_VERBOSE, _("Rejecting `%s'.\n"),
		     escnonprint (f->name));
	  f = delelement (f, &start);
	}
      else
	f = f->next;
    }
  /* Now weed out the files that do not match our globbing pattern.
     If we are dealing with a globbing pattern, that is.  */
  if (*u->file && (action == GLOB_GLOBALL || action == GLOB_GETONE))
    {
      int matchres = 0;

      f = start;
      while (f)
	{
	  matchres = fnmatch (u->file, f->name, 0);
	  if (matchres == -1)
	    {
	      logprintf (LOG_NOTQUIET, "%s: %s\n", con->target,
			 strerror (errno));
	      break;
	    }
	  if (matchres == FNM_NOMATCH)
	    f = delelement (f, &start); /* delete the element from the list */
	  else
	    f = f->next;        /* leave the element in the list */
	}
      if (matchres == -1)
	{
	  freefileinfo (start);
	  return RETRBADPATTERN;
	}
    }
  if (start)
    {
      /* Just get everything.  */
      ftp_retrieve_list (u, start, con);
    }
  else if (!start)
    {
      if (action == GLOB_GLOBALL)
	{
	  /* No luck.  */
	  /* #### This message SUCKS.  We should see what was the
	     reason that nothing was retrieved.  */
	  logprintf (LOG_VERBOSE, _("No matches on pattern `%s'.\n"),
		     escnonprint (u->file));
	}
      else /* GLOB_GETONE or GLOB_GETALL */
	{
	  /* Let's try retrieving it anyway.  */
	  con->st |= ON_YOUR_OWN;
	  res = ftp_loop_internal (u, NULL, con);
	  return res;
	}
    }
  freefileinfo (start);
  if (opt.quota && total_downloaded_bytes > opt.quota)
    return QUOTEXC;
  else
    /* #### Should we return `res' here?  */
    return RETROK;
}

/* The wrapper that calls an appropriate routine according to contents
   of URL.  Inherently, its capabilities are limited on what can be
   encoded into a URL.  */
uerr_t
ftp_loop (struct url *u, int *dt, struct url *proxy)
{
  ccon con;			/* FTP connection */
  uerr_t res;

  *dt = 0;

  xzero (con);

  con.csock = -1;
  con.st = ON_YOUR_OWN;
  con.rs = ST_UNIX;
  con.id = NULL;
  con.proxy = proxy;

  /* If the file name is empty, the user probably wants a directory
     index.  We'll provide one, properly HTML-ized.  Unless
     opt.htmlify is 0, of course.  :-) */
  if (!*u->file && !opt.recursive)
    {
      struct fileinfo *f;
      res = ftp_get_listing (u, &con, &f);

      if (res == RETROK)
	{
	  if (opt.htmlify && !opt.spider)
	    {
	      char *filename = (opt.output_document
				? xstrdup (opt.output_document)
				: (con.target ? xstrdup (con.target)
				   : url_file_name (u)));
	      res = ftp_index (filename, u, f);
	      if (res == FTPOK && opt.verbose)
		{
		  if (!opt.output_document)
		    {
		      struct_stat st;
		      wgint sz;
		      if (stat (filename, &st) == 0)
			sz = st.st_size;
		      else
			sz = -1;
		      logprintf (LOG_NOTQUIET,
				 _("Wrote HTML-ized index to `%s' [%s].\n"),
				 filename, number_to_static_string (sz));
		    }
		  else
		    logprintf (LOG_NOTQUIET,
			       _("Wrote HTML-ized index to `%s'.\n"),
			       filename);
		}
	      xfree (filename);
	    }
	  freefileinfo (f);
	}
    }
  else
    {
      int ispattern = 0;
      if (opt.ftp_glob)
	{
	  /* Treat the URL as a pattern if the file name part of the
	     URL path contains wildcards.  (Don't check for u->file
	     because it is unescaped and therefore doesn't leave users
	     the option to escape literal '*' as %2A.)  */
	  char *file_part = strrchr (u->path, '/');
	  if (!file_part)
	    file_part = u->path;
	  ispattern = has_wildcards_p (file_part);
	}
      if (ispattern || opt.recursive || opt.timestamping)
	{
	  /* ftp_retrieve_glob is a catch-all function that gets called
	     if we need globbing, time-stamping or recursion.  Its
	     third argument is just what we really need.  */
	  res = ftp_retrieve_glob (u, &con,
				   ispattern ? GLOB_GLOBALL : GLOB_GETONE);
	}
      else
	res = ftp_loop_internal (u, NULL, &con);
    }
  if (res == FTPOK)
    res = RETROK;
  if (res == RETROK)
    *dt |= RETROKF;
  /* If a connection was left, quench it.  */
  if (con.csock != -1)
    fd_close (con.csock);
  xfree_null (con.id);
  con.id = NULL;
  xfree_null (con.target);
  con.target = NULL;
  return res;
}

/* Delete an element from the fileinfo linked list.  Returns the
   address of the next element, or NULL if the list is exhausted.  It
   can modify the start of the list.  */
static struct fileinfo *
delelement (struct fileinfo *f, struct fileinfo **start)
{
  struct fileinfo *prev = f->prev;
  struct fileinfo *next = f->next;

  xfree (f->name);
  xfree_null (f->linkto);
  xfree (f);

  if (next)
    next->prev = prev;
  if (prev)
    prev->next = next;
  else
    *start = next;
  return next;
}

/* Free the fileinfo linked list of files.  */
static void
freefileinfo (struct fileinfo *f)
{
  while (f)
    {
      struct fileinfo *next = f->next;
      xfree (f->name);
      if (f->linkto)
	xfree (f->linkto);
      xfree (f);
      f = next;
    }
}


text:
package net.amham.odfe.xpath;

import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;
import java.util.logging.Logger;

import org.odftoolkit.odfdom.dom.element.text.TextParagraphElementBase;
import org.odftoolkit.odfdom.dom.element.text.TextSectionElement;
import org.odftoolkit.simple.text.Paragraph;
import org.w3c.dom.NodeList;

import net.amham.odfe.difference.DocumentDifference;

public class XPathSectionDifference extends XPathDifference {

	private final static   Logger LOGGER = Logger.getLogger(XPathSectionDifference.class.getName());

	public XPathSectionDifference() {
		type = Type.SECTION;
	}

	public List <DocumentDifference>  getDocumentDifferences() {
		//look for style nodes
		List <DocumentDifference> retDiffs = new ArrayList<DocumentDifference>();
		if (newNode != null) {
			DocumentDifference diff = new DocumentDifference(DocumentDifference.DiffType.CHANGED);
			String paraStyle = original.getParent().getAttributeValue("style:name");
			diff.setName(paraStyle);
			String detail = null;
			for (DocumentDifference attDiff : original.getAttributeDifferences(newNode)) {
				detail += attDiff.getName() + " : " + attDiff.getDetail();
			}
			diff.setDetail(detail);
			retDiffs.add(diff);
		}
		return retDiffs;
	}

	public String getTarget() {
		String paraStyle = original.getParent().getParent().getParent().getAttributeValue("style:name");
		return "//text:section[@text:style-name='" + paraStyle + "']";
	}

	public String commentNodes(NodeList nodeList) {
		String added = null;
	    for (int i = 0; null!=nodeList && i < nodeList.getLength(); i++) {
	    	TextSectionElement section = (TextSectionElement) nodeList.item(i);
	    	NodeList paraNodeList = section.getElementsByTagName("text:p");
	    	if(null!=paraNodeList) {
		    	TextParagraphElementBase para = (TextParagraphElementBase) paraNodeList.item(0);
				Paragraph para1 = Paragraph.getInstanceof(para);
				String detail = "";
				if (newNode != null) {
					for (DocumentDifference attDiff : original.getAttributeDifferences(newNode)) {
						detail += attDiff.getName() + " : " + attDiff.getDetail() + "\n";
					}
				}
				para1.addComment(detail, "ODFExplorer");
				added = "Section Change:\n" + detail;
	    	}
	    }
	    return added;
	}

	public XPathNode findNewNode() {
		XPathNode sibling = null;
		// for section columns the node line up in order
		//so this must be the first original
		//therefore the first NEW should be the corresponding node
		Iterator<List<XPathNode>> sibs = original.getParent().getChildren().iterator();
		boolean notFound = true;
		if (sibs.hasNext()) {
			List<XPathNode> siblings = sibs.next();
			Iterator<XPathNode> xpi = siblings.iterator();
			while (xpi.hasNext() && notFound) {
				sibling = xpi.next();
				if ( sibling.getState() == XPathNode.State.NEW )
				{
					notFound = false;
					LOGGER.info("MATCH column section node");
					StringBuilder atts = new StringBuilder();
					for (String att : original.getAttributeNames()) {
						atts.append(att);
						atts.append(":");
						atts.append(original.getAttributeValue(att));
					}
					LOGGER.info("Orignal " + atts.toString());
					StringBuilder sibatts = new StringBuilder();
					for (String att : sibling.getAttributeNames()) {
						sibatts.append(att);
						sibatts.append(":");
						sibatts.append(sibling.getAttributeValue(att));
					}
					LOGGER.info("Sibling " + sibatts.toString());

				}
				else {
					sibling = null;
				}
			}
		}
		if(sibling == null) {
			LOGGER.info("Unable to find a NEW column section node");
		}
		return sibling;
	}

//	protected boolean compareNamedSibling(XPathNode sibling) {
//		LOGGER.info("compareNamedSibling - returning false");
//
//
//			//is this a match for a renamed auto style?
//
//			//problem here is we are not sure if these really are different instances
//			//of the same xpath -
//			//need to be node type specific - case just looked at was for sections
///*			LOGGER.info("Remove " + sibling.getName() + " " + sibling.getState() + " "
//					+ diffDetails);
//			notFound = false;
//			newNodeToDelete = sibling;
//			LOGGER.info("diff found for " + state.toString() + " " + nodePath);
//			state = XPathNode.State.DIFF;
//			for (DocumentDifference attDiff : getAttributeDifferences(sibling)) {
//				diffDetails += attDiff.getName() + " : " + attDiff.getDetail() + " ";
//			}
//			xPathDiff.addSibling(sibling);
//		}*/
//
//		return false;
//
//	}

}

redpajama_set_name: RedPajamaGithub


text:
// Copyright 2015 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

#include "content/renderer/media/media_recorder_handler.h"

#include <utility>

#include "base/bind.h"
#include "base/location.h"
#include "base/logging.h"
#include "base/macros.h"
#include "base/strings/string_tokenizer.h"
#include "base/strings/string_util.h"
#include "content/renderer/media/audio_track_recorder.h"
#include "content/renderer/media/media_stream_audio_track.h"
#include "content/renderer/media/media_stream_track.h"
#include "content/renderer/media/video_track_recorder.h"
#include "content/renderer/media/webrtc_uma_histograms.h"
#include "media/audio/audio_parameters.h"
#include "media/base/audio_bus.h"
#include "media/base/bind_to_current_loop.h"
#include "media/base/mime_util.h"
#include "media/base/video_frame.h"
#include "media/muxers/webm_muxer.h"
#include "third_party/WebKit/public/platform/WebMediaRecorderHandlerClient.h"
#include "third_party/WebKit/public/platform/WebString.h"

using base::TimeDelta;
using base::TimeTicks;

namespace content {

MediaRecorderHandler::MediaRecorderHandler()
    : video_bits_per_second_(0),
      audio_bits_per_second_(0),
      use_vp9_(false),
      recording_(false),
      client_(nullptr),
      weak_factory_(this) {}

MediaRecorderHandler::~MediaRecorderHandler() {
  DCHECK(main_render_thread_checker_.CalledOnValidThread());
  // Send a |last_in_slice| to our |client_|.
  if (client_)
    client_->writeData(nullptr, 0u, true);
}

bool MediaRecorderHandler::canSupportMimeType(
    const blink::WebString& web_type, const blink::WebString& web_codecs) {
  DCHECK(main_render_thread_checker_.CalledOnValidThread());
  // An empty |web_type| means MediaRecorderHandler can choose its preferred
  // codecs.
  if (web_type.isEmpty())
    return true;

  const std::string type(web_type.utf8());
  const bool video = base::EqualsCaseInsensitiveASCII(type, "video/webm");
  const bool audio =
      video ? false : base::EqualsCaseInsensitiveASCII(type, "audio/webm");
  if (!video && !audio)
    return false;

  // Both |video| and |audio| support empty |codecs|; |type| == "video" supports
  // vp8, vp9 or opus; |type| = "audio", supports only opus.
  // http://www.webmproject.org/docs/container Sec:"HTML5 Video Type Parameters"
  static const char* kVideoCodecs[] = { "vp8", "vp9", "opus" };
  static const char* kAudioCodecs[] = { "opus" };
  const char** codecs = video ? &kVideoCodecs[0] : &kAudioCodecs[0];
  int codecs_count =  video ? arraysize(kVideoCodecs) : arraysize(kAudioCodecs);

  std::vector<std::string> codecs_list;
  media::ParseCodecString(web_codecs.utf8(), &codecs_list, true /* strip */);
  for (const auto& codec : codecs_list) {
    const auto found = std::find_if(
        &codecs[0], &codecs[codecs_count], [&codec](const char* name) {
          return base::EqualsCaseInsensitiveASCII(codec, name);
        });
    if (found == &codecs[codecs_count])
      return false;
  }
  return true;
}

bool MediaRecorderHandler::initialize(
    blink::WebMediaRecorderHandlerClient* client,
    const blink::WebMediaStream& media_stream,
    const blink::WebString& type,
    const blink::WebString& codecs,
    int32_t audio_bits_per_second,
    int32_t video_bits_per_second) {
  DCHECK(main_render_thread_checker_.CalledOnValidThread());
  // Save histogram data so we can see how much MediaStream Recorder is used.
  // The histogram counts the number of calls to the JS API.
  UpdateWebRTCMethodCount(WEBKIT_MEDIA_STREAM_RECORDER);

  if (!canSupportMimeType(type, codecs)) {
    DLOG(ERROR) << "Can't support " << type.utf8()
                << ";codecs=" << codecs.utf8();
    return false;
  }
  use_vp9_ = base::ToLowerASCII(codecs.utf8()).find("vp9") != std::string::npos;
  media_stream_ = media_stream;
  DCHECK(client);
  client_ = client;

  audio_bits_per_second_ = audio_bits_per_second;
  video_bits_per_second_ = video_bits_per_second;
  return true;
}

bool MediaRecorderHandler::start() {
  DCHECK(main_render_thread_checker_.CalledOnValidThread());
  DCHECK(!recording_);
  return start(0);
}

bool MediaRecorderHandler::start(int timeslice) {
  DCHECK(main_render_thread_checker_.CalledOnValidThread());
  DCHECK(!recording_);
  DCHECK(!media_stream_.isNull());
  DCHECK(timeslice_.is_zero());
  DCHECK(!webm_muxer_);

  timeslice_ = TimeDelta::FromMilliseconds(timeslice);
  slice_origin_timestamp_ = TimeTicks::Now();

  blink::WebVector<blink::WebMediaStreamTrack> video_tracks, audio_tracks;
  media_stream_.videoTracks(video_tracks);
  media_stream_.audioTracks(audio_tracks);

  if (video_tracks.isEmpty() && audio_tracks.isEmpty()) {
    LOG(WARNING) << __FUNCTION__ << ": no media tracks.";
    return false;
  }

  const bool use_audio_tracks =
      !audio_tracks.isEmpty() && MediaStreamAudioTrack::From(audio_tracks[0]);

  webm_muxer_.reset(new media::WebmMuxer(
      use_vp9_ ? media::kCodecVP9 : media::kCodecVP8,
      video_tracks.size() > 0, use_audio_tracks,
      base::Bind(&MediaRecorderHandler::WriteData,
                 weak_factory_.GetWeakPtr())));

  if (!video_tracks.isEmpty()) {
    // TODO(mcasas): The muxer API supports only one video track. Extend it to
    // several video tracks, see http://crbug.com/528523.
    LOG_IF(WARNING, video_tracks.size() > 1u)
        << "Recording multiple video tracks is not implemented. "
        << "Only recording first video track.";
    const blink::WebMediaStreamTrack& video_track = video_tracks[0];
    if (video_track.isNull())
      return false;

    const VideoTrackRecorder::OnEncodedVideoCB on_encoded_video_cb =
        media::BindToCurrentLoop(base::Bind(
            &MediaRecorderHandler::OnEncodedVideo, weak_factory_.GetWeakPtr()));

    video_recorders_.push_back(new VideoTrackRecorder(
        use_vp9_, video_track, on_encoded_video_cb, video_bits_per_second_));
  }

  if (use_audio_tracks) {
    // TODO(ajose): The muxer API supports only one audio track. Extend it to
    // several tracks.
    LOG_IF(WARNING, audio_tracks.size() > 1u)
        << "Recording multiple audio"
        << " tracks is not implemented.  Only recording first audio track.";
    const blink::WebMediaStreamTrack& audio_track = audio_tracks[0];
    if (audio_track.isNull())
      return false;

    const AudioTrackRecorder::OnEncodedAudioCB on_encoded_audio_cb =
        media::BindToCurrentLoop(base::Bind(
            &MediaRecorderHandler::OnEncodedAudio, weak_factory_.GetWeakPtr()));

    audio_recorders_.push_back(new AudioTrackRecorder(
        audio_track, on_encoded_audio_cb, audio_bits_per_second_));
  }

  recording_ = true;
  return true;
}

void MediaRecorderHandler::stop() {
  DCHECK(main_render_thread_checker_.CalledOnValidThread());
  // Don't check |recording_| since we can go directly from pause() to stop().

  recording_ = false;
  timeslice_ = TimeDelta::FromMilliseconds(0);
  video_recorders_.clear();
  audio_recorders_.clear();
  webm_muxer_.reset();
}

void MediaRecorderHandler::pause() {
  DCHECK(main_render_thread_checker_.CalledOnValidThread());
  DCHECK(recording_);
  recording_ = false;
  for (const auto& video_recorder : video_recorders_)
    video_recorder->Pause();
}

void MediaRecorderHandler::resume() {
  DCHECK(main_render_thread_checker_.CalledOnValidThread());
  DCHECK(!recording_);
  recording_ = true;
  for (const auto& video_recorder : video_recorders_)
    video_recorder->Resume();
}

void MediaRecorderHandler::OnEncodedVideo(
    const scoped_refptr<media::VideoFrame>& video_frame,
    scoped_ptr<std::string> encoded_data,
    TimeTicks timestamp,
    bool is_key_frame) {
  DCHECK(main_render_thread_checker_.CalledOnValidThread());
  if (!webm_muxer_)
    return;
  webm_muxer_->OnEncodedVideo(video_frame, std::move(encoded_data), timestamp,
                              is_key_frame);
}

void MediaRecorderHandler::OnEncodedAudio(const media::AudioParameters& params,
                                          scoped_ptr<std::string> encoded_data,
                                          base::TimeTicks timestamp) {
  DCHECK(main_render_thread_checker_.CalledOnValidThread());
  if (webm_muxer_)
    webm_muxer_->OnEncodedAudio(params, std::move(encoded_data), timestamp);
}

void MediaRecorderHandler::WriteData(base::StringPiece data) {
  DCHECK(main_render_thread_checker_.CalledOnValidThread());
  // Non-buffered mode does not need to check timestamps.
  if (timeslice_.is_zero()) {
    client_->writeData(data.data(), data.length(), true  /* lastInSlice */);
    return;
  }

  const TimeTicks now = TimeTicks::Now();
  const bool last_in_slice = now > slice_origin_timestamp_ + timeslice_;
  DVLOG_IF(1, last_in_slice) << "Slice finished @ " << now;
  if (last_in_slice)
    slice_origin_timestamp_ = now;
  client_->writeData(data.data(), data.length(), last_in_slice);
}

void MediaRecorderHandler::OnVideoFrameForTesting(
    const scoped_refptr<media::VideoFrame>& frame,
    const TimeTicks& timestamp) {
  for (auto* recorder : video_recorders_)
    recorder->OnVideoFrameForTesting(frame, timestamp);
}

void MediaRecorderHandler::OnAudioBusForTesting(
    const media::AudioBus& audio_bus,
    const base::TimeTicks& timestamp) {
  for (auto* recorder : audio_recorders_)
    recorder->OnData(audio_bus, timestamp);
}

void MediaRecorderHandler::SetAudioFormatForTesting(
    const media::AudioParameters& params) {
  for (auto* recorder : audio_recorders_)
    recorder->OnSetFormat(params);
}

}  // namespace content


text:
const debug = require(`debug`)(`contentful-text-search:indexer`)
const transform = require(`./transform`)
const config = require(`./index-config`)

module.exports = class Indexer {
  constructor(space, contentfulClient, elasticsearchClient) {
    this.space = space
    this.contentful = contentfulClient
    this.elasticsearch = elasticsearchClient
    this.queryGeneratedSinceLastIndexChange = false
  }

  /*
    Delete and recreate an index for each locale in the space, and index the content into these indices.
  */
  async fullReindex() {
    try {
      const { entries, contentTypes, locales } = await this.getFormattedData()
      // recreate an index for each locale and upload entries into it
      const recreateIndicesAndUploadEntries = locales.map(async localeObj => {
        const locale = localeObj.code
        const index = `contentful_${this.space}_${locale.toLowerCase()}`
        const indexConfig = config.createIndexConfig(contentTypes, locale)
        await this.elasticsearch.recreateIndex(index, indexConfig)
        await this.indexContent(entries, locale, index)
      })
      await Promise.all(recreateIndicesAndUploadEntries)
    } catch (err) {
      debug(`Error in fullReindex: %s`, err)
      throw new Error(err)
    }
  }

  /*
    Clear the indices and reindex all the content from Contentful
  */
  async reindexContent() {
    try {
      const { entries, locales } = await this.getFormattedData()
      // upload entries into a different index for each locale
      const clearIndicesAndUploadEntries = locales.map(async localeObj => {
        const locale = localeObj.code
        const index = `contentful_${this.space}_${locale.toLowerCase()}`
        await this.elasticsearch.clearIndex(index)
        await this.indexContent(entries, locale, index)
      })
      await Promise.all(clearIndicesAndUploadEntries)
    } catch (err) {
      debug(`Error in reindexContent: %s`, err)
      throw new Error(err)
    }
  }

  /*
    Delete all indices related to a contentful space
  */
  async deleteAllIndices() {
    // TODO: delete contentful_space_* in case a locale is no longer in that space
    const { locales } = await this.contentful.client.getSpace()
    const deleteIndices = locales.map(async locale => {
      const index = `contentful_${this.space}_${locale.code.toLowerCase()}`
      try {
        await this.elasticsearch.client.indices.delete({
          index,
        })
      } catch (err) {
        // catch but don't throw in case the index doesn't exist
        debug(`Could not delete index %s`, index)
      }
    })
    await Promise.all(deleteIndices)
  }

  // Retrieve data from contentful and reformat it
  async getFormattedData(rawEntries) {
    try {
      let resolvedEntries
      if (rawEntries) {
        resolvedEntries = await this.contentful.resolveReferences(rawEntries)
      } else {
        resolvedEntries = await this.contentful.getResolvedEntries()
      }
      const contentTypesResponse = await this.contentful.client.getContentTypes()
      const { locales } = await this.contentful.client.getSpace()
      const contentTypes = transform.reduceContentTypes(
        contentTypesResponse.items
      )
      const entries = transform.reformatEntries(
        resolvedEntries,
        contentTypes,
        locales
      )
      return { entries, locales, contentTypes }
    } catch (err) {
      debug(`Error getting formatted data: %s`, err)
      throw new Error(err)
    }
  }

  // Upload entries into a given index
  async indexContent(entries, locale, index) {
    try {
      const payload = transform.generatePayload(entries, locale, index)
      if (payload.body.length > 0) {
        await this.elasticsearch.client.bulk(payload)
      }
    } catch (err) {
      // don't throw since we should continue indexing for other locales
      debug(`Could not index entries for ${locale}`)
    }
  }

  async indexSingleEntry(entry) {
    try {
      const { entries, locales } = await this.getFormattedData([entry])
      // index an entry for each locale
      const indexLocalisedEntries = locales.map(async localeObj => {
        const locale = localeObj.code
        const index = `contentful_${this.space}_${locale.toLowerCase()}`
        await this.indexContent(entries, locale, index)
      })
      await Promise.all(indexLocalisedEntries)
    } catch (err) {
      debug(`Error in indexSingleEntry: %s`, err)
      throw new Error(err)
    }
  }

  async deleteSingleEntry(entry) {
    try {
      const { locales } = await this.contentful.client.getSpace()
      // delete the entry for each locale
      const deleteLocalisedEntries = locales.map(async localeObj => {
        const locale = localeObj.code
        const index = `contentful_${this.space}_${locale.toLowerCase()}`
        try {
          await this.elasticsearch.client.delete({
            index,
            type: entry.sys.contentType.sys.id,
            id: entry.sys.id,
          })
        } catch (err) {
          // don't throw error because the entry may not exist
          debug(`Failed to delete entry for locale ${locale}`)
        }
      })
      await Promise.all(deleteLocalisedEntries)
    } catch (err) {
      debug(`Error in deleteSingleEntry: %s`, err)
      throw new Error(err)
    }
  }
}


text:
package usn

// A Processor is capable of processing USN change journal records.
type Processor func(Record)

// Process runs p(r) if p is non-nil.
func (p Processor) Process(r Record) {
	if p != nil {
		p(r)
	}
}


text:
define(["require", "exports"], function (require, exports) {
    /** An instance of EventGroup allows anything with a handle to it to trigger events on it.
     *  If the target is an HTMLElement, the event will be attached to the element and can be
     *  triggered as usual (like clicking for onclick).
     *  The event can be triggered by calling EventGroup.raise() here. If the target is an
     *  HTMLElement, the event gets raised and is handled by the browser. Otherwise, it gets
     *  handled here in EventGroup, and the handler is called in the context of the parent
     *  (which is passed in in the constructor).
     */
    var EventGroup = (function () {
        /** parent: the context in which events attached to non-HTMLElements are called */
        function EventGroup(parent) {
            this._id = EventGroup._uniqueId++;
            this._parent = parent;
            this._eventRecords = [];
        }
        /** For IE8, bubbleEvent is ignored here and must be dealt with by the handler.
         *  Events raised here by default have bubbling set to false and cancelable set to true.
         *  This applies also to built-in events being raised manually here on HTMLElements,
         *  which may lead to unexpected behavior if it differs from the defaults.
         */
        EventGroup.raise = function (target, eventName, eventArgs, bubbleEvent) {
            var retVal;
            if (EventGroup._isElement(target)) {
                if (document.createEvent) {
                    var ev = document.createEvent('HTMLEvents');
                    ev.initEvent(eventName, bubbleEvent, true);
                    ev['args'] = eventArgs;
                    retVal = target.dispatchEvent(ev);
                }
                else if (document['createEventObject']) {
                    var evObj = document['createEventObject'](eventArgs);
                    // cannot set cancelBubble on evObj, fireEvent will overwrite it
                    target.fireEvent("on" + eventName, evObj);
                }
            }
            else {
                while (target && retVal !== false) {
                    var eventRecords = target.__events__ ? target.__events__[eventName] : null;
                    for (var id in eventRecords) {
                        var eventRecordList = eventRecords[id];
                        for (var listIndex = 0; retVal !== false && listIndex < eventRecordList.length; listIndex++) {
                            var record = eventRecordList[listIndex];
                            // Call the callback in the context of the parent, using the supplied eventArgs.
                            retVal = record.callback.call(record.parent, eventArgs);
                        }
                    }
                    // If the target has a parent, bubble the event up.
                    target = bubbleEvent ? target.parent : null;
                }
            }
            return retVal;
        };
        EventGroup.isObserved = function (target, eventName) {
            return !!(target && target.__events__ && target.__events__[eventName]);
        };
        /** Check to see if the target has declared support of the given event. */
        EventGroup.isDeclared = function (target, eventName) {
            return !!(target && target.__declaredEvents && target.__declaredEvents[eventName]);
        };
        EventGroup.stopPropagation = function (event) {
            if (event.stopPropagation) {
                event.stopPropagation();
            }
            else {
                event.cancelBubble = true;
            }
        };
        EventGroup._isElement = function (target) {
            return !!target && (target instanceof HTMLElement || (target.dispatchEvent && target.addEventListener));
        };
        EventGroup.prototype.dispose = function () {
            this.off();
            this._parent = null;
        };
        /** On the target, attach a set of events, where the events object is a name to function mapping. */
        EventGroup.prototype.onAll = function (target, events, useCapture) {
            for (var eventName in events) {
                this.on(target, eventName, events[eventName], useCapture);
            }
        };
        /** On the target, attach an event whose handler will be called in the context of the parent
         * of this instance of EventGroup.
         */
        EventGroup.prototype.on = function (target, eventName, callback, useCapture) {
            if (eventName.indexOf(',') > -1) {
                var events = eventName.split(/[ ,]+/);
                for (var i = 0; i < events.length; i++) {
                    this.on(target, events[i], callback, useCapture);
                }
            }
            else {
                var parent = this._parent;
                var eventRecord = {
                    target: target,
                    eventName: eventName,
                    parent: parent,
                    callback: callback,
                    elementCallback: null,
                    useCapture: useCapture
                };
                // Initialize and wire up the record on the target, so that it can call the callback if the event fires.
                target.__events__ = target.__events__ || {};
                target.__events__[eventName] = target.__events__[eventName] || {
                    count: 0
                };
                target.__events__[eventName][this._id] = target.__events__[eventName][this._id] || [];
                target.__events__[eventName][this._id].push(eventRecord);
                target.__events__[eventName].count++;
                function _processElementEvent() {
                    try {
                        var result = callback.apply(parent, arguments);
                        if (result === false && arguments[0] && arguments[0].preventDefault) {
                            var e = arguments[0];
                            e.preventDefault();
                            e.cancelBubble = true;
                        }
                    }
                    catch (e) {
                    }
                    return result;
                }
                if (EventGroup._isElement(target)) {
                    eventRecord.elementCallback = _processElementEvent;
                    if (target.addEventListener) {
                        /* tslint:disable:ban-native-functions */
                        target.addEventListener(eventName, _processElementEvent, useCapture);
                    }
                    else if (target.attachEvent) {
                        target.attachEvent("on" + eventName, _processElementEvent);
                    }
                }
                // Remember the record locally, so that it can be removed.
                this._eventRecords.push(eventRecord);
            }
        };
        EventGroup.prototype.off = function (target, eventName, callback, useCapture) {
            for (var i = 0; i < this._eventRecords.length; i++) {
                var eventRecord = this._eventRecords[i];
                if ((!target || target === eventRecord.target) &&
                    (!eventName || eventName === eventRecord.eventName) &&
                    (!callback || callback === eventRecord.callback) &&
                    ((typeof useCapture !== 'boolean') || useCapture === eventRecord.useCapture)) {
                    var targetArrayLookup = eventRecord.target.__events__[eventRecord.eventName];
                    var targetArray = targetArrayLookup ? targetArrayLookup[this._id] : null;
                    // We may have already target's entries, so check for null.
                    if (targetArray) {
                        if (targetArray.length === 1 || !callback) {
                            targetArrayLookup.count -= targetArray.length;
                            delete eventRecord.target.__events__[eventRecord.eventName][this._id];
                        }
                        else {
                            targetArrayLookup.count--;
                            targetArray.splice(targetArray.indexOf(eventRecord), 1);
                        }
                        if (!targetArrayLookup.count) {
                            delete eventRecord.target.__events__[eventRecord.eventName];
                        }
                    }
                    if (eventRecord.elementCallback) {
                        if (eventRecord.target.removeEventListener) {
                            eventRecord.target.removeEventListener(eventRecord.eventName, eventRecord.elementCallback, eventRecord.useCapture);
                        }
                        else if (eventRecord.target.detachEvent) {
                            eventRecord.target.detachEvent("on" + eventRecord.eventName, eventRecord.elementCallback);
                        }
                    }
                    this._eventRecords.splice(i--, 1);
                }
            }
        };
        /** Trigger the given event in the context of this instance of EventGroup. */
        EventGroup.prototype.raise = function (eventName, eventArgs, bubbleEvent) {
            return EventGroup.raise(this._parent, eventName, eventArgs, bubbleEvent);
        };
        /** Declare an event as being supported by this instance of EventGroup. */
        EventGroup.prototype.declare = function (event) {
            var declaredEvents = this._parent.__declaredEvents = this._parent.__declaredEvents || {};
            if (typeof event === 'string') {
                declaredEvents[event] = true;
            }
            else {
                for (var i = 0; i < event.length; i++) {
                    declaredEvents[event[i]] = true;
                }
            }
        };
        EventGroup._uniqueId = 0;
        return EventGroup;
    })();
    exports.default = EventGroup;
});


text:
(function() {

    'use strict';

    angular
        .module('cometApp')
        .factory('callService', callService);

    callService.$inject = ['$log', '$http', 'authService'];

    function callService ($log, $http, authService) {

        var parentUrl = '/projects/',
            resourceUrl = '/calls/';

        return {
            create: create,
            update: update,
            getById: getById,
            updateMembers: updateMembers,
            updateSummary: updateSummary
        };

        /**
         * @name create
         * @desc creates a new call
         */
        function create (projectId, channelId, call) {
          var url = getBaseUrl(projectId, channelId);
          return $http.post(url, call, authService.getJwtHeader());
        }

        /**
         * @name getById
         * @desc returns a call by id
         */
        function getById (projectId, channelId, callId) {
          var url = getBaseUrl(projectId, channelId) + callId;
          return $http.get(url, authService.getJwtHeader());
        }

        /**
         * @name update
         * @desc update call
         */
        function update (projectId, channelId, call) {
          var url = getBaseUrl(projectId, channelId) + call.id;
          return $http.put(url, call, authService.getJwtHeader());
        }

        /**
         * @name updateMembers
         * @desc update call members
         */
        function updateMembers (projectId, channelId, callId) {
          var url = getBaseUrl(projectId, channelId) + callId + '/members';
          return $http.put(url, { test : '' }, authService.getJwtHeader());
        }

        /**
         * @name updateSummary
         * @desc update call summary
         */
        function updateSummary (projectId, channelId, call) {
          var url = getBaseUrl(projectId, channelId) + call.id + '/summary';
          return $http.put(url, { summary : call.summary }, authService.getJwtHeader());
        }

        /**
         * @name getbaseUrl
         * @desc return the base url for the channels resource
         */
        function getBaseUrl(projectId, channelId) {
          //'/projects/:project_id/channels/:channel_id/calls'
          return parentUrl + projectId + '/channels/' + channelId + resourceUrl;
        }
    }
})();


text:
<Type Name="AmdDrawBuffersBlend" FullName="MonoMac.OpenGL.AmdDrawBuffersBlend">
  <TypeSignature Language="C#" Value="public enum AmdDrawBuffersBlend" />
  <TypeSignature Language="ILAsm" Value=".class public auto ansi sealed AmdDrawBuffersBlend extends System.Enum" />
  <AssemblyInfo>
    <AssemblyName>MonoMac</AssemblyName>
    <AssemblyVersion>0.0.0.0</AssemblyVersion>
  </AssemblyInfo>
  <Base>
    <BaseTypeName>System.Enum</BaseTypeName>
  </Base>
  <Docs>
    <summary>To be added.</summary>
    <remarks>To be added.</remarks>
  </Docs>
  <Members />
</Type>


text:
package esidev

/*
Unprocessable entity */
type GetMarketsRegionIdHistoryUnprocessableEntity struct {
	/*
	 Unprocessable entity message */
	Error_ string `json:"error,omitempty"`
}


text:
#pragma once

#include <thrust/detail/config.h>
#include <thrust/system/omp/detail/execution_policy.h>

namespace thrust
{
namespace system
{
namespace omp
{
namespace detail
{

template<typename DerivedPolicy,
         typename ForwardIterator,
         typename Predicate>
  ForwardIterator remove_if(execution_policy<DerivedPolicy> &exec,
                            ForwardIterator first,
                            ForwardIterator last,
                            Predicate pred);

template<typename DerivedPolicy,
         typename ForwardIterator,
         typename InputIterator,
         typename Predicate>
  ForwardIterator remove_if(execution_policy<DerivedPolicy> &exec,
                            ForwardIterator first,
                            ForwardIterator last,
                            InputIterator stencil,
                            Predicate pred);

template<typename DerivedPolicy,
         typename InputIterator,
         typename OutputIterator,
         typename Predicate>
  OutputIterator remove_copy_if(execution_policy<DerivedPolicy> &exec,
                                InputIterator first,
                                InputIterator last,
                                OutputIterator result,
                                Predicate pred);

template<typename DerivedPolicy,
         typename InputIterator1,
         typename InputIterator2,
         typename OutputIterator,
         typename Predicate>
  OutputIterator remove_copy_if(execution_policy<DerivedPolicy> &exec,
                                InputIterator1 first,
                                InputIterator1 last,
                                InputIterator2 stencil,
                                OutputIterator result,
                                Predicate pred);

} // end namespace detail
} // end namespace omp
} // end namespace system
} // end namespace thrust

#include <thrust/system/omp/detail/remove.inl>