text:
=head1 LICENSE

Copyright [1999-2015] Wellcome Trust Sanger Institute and the EMBL-European Bioinformatics Institute
Copyright [2016-2022] EMBL-European Bioinformatics Institute

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

     http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.

=cut

package EnsEMBL::Web::Document::HTML::FTPtable;

### This module outputs a table of links to the FTP site

use strict;

use HTML::Entities qw(encode_entities);

use EnsEMBL::Web::Document::Table;

use base qw(EnsEMBL::Web::Document::HTML);

sub render {
  my $self            = shift;
  my $hub             = $self->hub;
  my $species_defs    = $hub->species_defs;
  my $division        = $species_defs->EG_DIVISION || '';

  my $html;

  if ($species_defs->ENSEMBL_MART_ENABLED) {
    $html .= qq(

Custom data sets

If you want to filter or customise your download, please try
Biomart, a web-based querying tool.

);
  }

  my $ftp = $division ? $species_defs->ENSEMBL_GENOMES_FTP_URL : $species_defs->ENSEMBL_FTP_URL;
  if ($species_defs->HAS_API_DOCS) {
    $html .= qq(

API Code

If you do not have access to git, you can obtain our latest API code as a gzipped tarball:

Download complete API for this release

Note: the API version needs to be the same as the databases you are accessing, so please
use git to obtain a previous version if querying older databases.

    );
  }

  my $mysql_dir = $division ? "$ftp/current/mysql/" : "$ftp/current_mysql/";

  unless ($species_defs->NO_PUBLIC_MYSQL) {
    $html .= qq(

Database dumps

Entire databases can be downloaded from our FTP site in a
variety of formats. Please be aware that some of these files
can run to many gigabytes of data.

Looking for MySQL dumps to install databases locally? See our
web installation instructions
for full details.

);
  }

(my $ftp_domain = $ftp) =~ s/\/pub//;

  $html .= qq(

Each directory on $ftp_domain contains a
README file, explaining the directory structure.

  );

  my $required_lookup = $self->required_types_for_species;
  my ($columns, $rows);

  my %title = (
    dna       => 'Masked and unmasked genome sequences associated with the assembly (contigs, chromosomes etc.)',
    cdna      => 'cDNA sequences for both Ensembl and "ab initio" predicted genes',
    cds       => 'Coding sequences for Ensembl or "ab initio" predicted genes',
    prot      => 'Protein sequences for Ensembl or "ab initio" predicted genes',
    rna       => 'Non-coding RNA gene predictions',
    embl      => 'Ensembl database dumps in EMBL nucleotide sequence database format',
    genbank   => 'Ensembl database dumps in GenBank nucleotide sequence database format',
    tsv       => 'External references in TSV format',
    rdf       => 'External references and other annotation data in RDF format',
    json      => 'External references and other annotation data in JSON format',
    gtf       => 'Gene sets for each species. These files include annotations of both coding and non-coding genes',
    gff3      => 'GFF3 provides access to all annotated transcripts which make up an Ensembl gene set',
    mysql     => 'All Ensembl MySQL databases are available in text format as are the SQL table definition files',
    emf       => 'Alignments of resequencing data from the ensembl_compara database',
    gvf       => 'Variation data in GVF format',
    vcf       => 'Variation data in VCF format',
    vep       => 'Cache files for use with the VEP script',
    funcgen   => 'Regulation data in GFF format',
    coll      => 'Additional regulation data (not in the database)',
    bed       => 'Constrained elements calculated using GERP',
    files     => 'Additional release data stored as flat files rather than MySQL for performance reasons',
    ancestral => 'Ancestral Allele data in FASTA format',
    bam       => 'Alignments against the genome',
  );

  $title{$_} = encode_entities($title{$_}) for keys %title;
  my $fave_title = $self->hub->species_defs->FAVOURITES_SYNONYM || 'Favourite';

  $columns = [
    { key => 'fave',    title => $fave_title,                    align => 'left',   width => '5%',  sort => 'html',
                        label => ''},
    { key => 'species', title => 'Species',                      align => 'left',   width => '10%', sort => 'html' },
    { key => 'dna',     title => 'DNA (FASTA)',                  align => 'center', width => '10%', sort => 'none' },
    { key => 'cdna',    title => 'cDNA (FASTA)',                 align => 'center', width => '10%', sort => 'none' },
    { key => 'cds',     title => 'CDS (FASTA)',                  align => 'center', width => '10%', sort => 'none' },
    { key => 'ncrna',   title => 'ncRNA (FASTA)',                align => 'center', width => '10%', sort => 'none' },
    { key => 'protseq', title => 'Protein sequence (FASTA)',     align => 'center', width => '10%', sort => 'none' },
    { key => 'embl',    title => 'Annotated sequence (EMBL)',    align => 'center', width => '10%', sort => 'none' },
    { key => 'genbank', title => 'Annotated sequence (GenBank)', align => 'center', width => '10%', sort => 'none' },
    { key => 'genes',   title => 'Gene sets',                    align => 'center', width => '10%', sort => 'none' },
    { key => 'xrefs',   title => 'Other annotations',            align => 'center', width => '10%', sort => 'none' },
  ];

  unless ($species_defs->NO_PUBLIC_MYSQL) {
    push @$columns,
    { key => 'mysql',   title => 'Whole databases',              align => 'center', width => '10%', sort => 'none' };
  }

  unless ($species_defs->NO_VARIATION) {
    push @$columns, (
    { key => 'var2',    title => 'Variation (GVF)',              align => 'center', width => '10%', sort => 'html' },
    { key => 'var4',    title => 'Variation (VCF)',              align => 'center', width => '10%', sort => 'html' },
    { key => 'var3',    title => 'Variation (VEP)',              align => 'center', width => '10%', sort => 'html' }
    );
  }

  ## No regulation dumps for non-vertebrates
  unless ($species_defs->NO_REGULATION || $species_defs->EG_DIVISION) {
    push @$columns, (
    { key => 'funcgen', title => 'Regulation (GFF)',             align => 'center', width => '10%', sort => 'html' },
    { key => 'files',   title => 'Data files',                   align => 'center', width => '10%', sort => 'html' },
    { key => 'bam',     title => 'BAM/BigWig',                          align => 'center', width => '10%', sort => 'html' },
    );
  }

  ## We want favourite species at the top of the table,
  ## then everything else alphabetically by display name
  my $all_species = [];
  my %fave_check = map {$_ => 1} @{$hub->get_favourite_species};
  foreach (@{$hub->get_favourite_species}) {
    push @$all_species, {
                          'url'         => $species_defs->get_config($_, 'SPECIES_URL'),
                          'dir'         => $species_defs->get_config($_, 'SPECIES_PRODUCTION_NAME'),
                          'display_name' => $species_defs->get_config($_, 'SPECIES_DISPLAY_NAME'),
                          'sci_name'    => $species_defs->get_config($_, 'SPECIES_SCIENTIFIC_NAME'),
                          'strain'      => $species_defs->get_config($_, 'SPECIES_STRAIN'),
                          'favourite'   => 1,
                        };
  }

  my @other_species;
  foreach ($species_defs->valid_species) {
    push @other_species, {
                          'url'         => $species_defs->get_config($_, 'SPECIES_URL'),
                          'dir'         => $species_defs->get_config($_, 'SPECIES_PRODUCTION_NAME'),
                          'display_name' => $species_defs->get_config($_, 'SPECIES_DISPLAY_NAME'),
                          'sci_name'    => $species_defs->get_config($_, 'SPECIES_SCIENTIFIC_NAME'),
                          'strain'      => $species_defs->get_config($_, 'SPECIES_STRAIN'),
                          'favourite'   => 0,
                        }
            unless $fave_check{$_};
  }
  push @$all_species, sort {$a->{'display_name'} cmp $b->{'display_name'}} @other_species;

  my ($ftp_base, $version, $rel) = $self->get_misc_params($ftp, $division);

  foreach my $sp (@$all_species) {
    my $sp_url    = $sp->{'url'};
    my $sp_name   = $sp->{'dir'};

    ## Add collection directory for relevant NV divisions
    my $dataset   = lc($hub->species_defs->get_config($sp_url, 'SPECIES_DATASET'));
    my $sp_dir    = ($dataset && $dataset ne $sp_name) ? $dataset.'_collection/' : '';
    $sp_dir      .= $sp_name;

    ## Vertebrate-specific - append scientific name if display name is common name
    my $display_name  = $sp->{'display_name'};
    my $sp_link       = sprintf('%s', $sp_url, $display_name);
    my $sci_name      = $sp->{'sci_name'};
    if ($hub->species_defs->USE_COMMON_NAMES && $display_name !~ /$sci_name/) {
      $sp_link .= "
 $sci_name";
    }

    my $sp_var    = $sp_dir. '_variation';
    my $databases = $hub->species_defs->get_config(ucfirst($sp_dir), 'databases');
    my $meta_info = $databases->{'DATABASE_VARIATION'}->{'meta_info'}->{1};
    my $has_vcf  = $meta_info && $meta_info->{'variation_source.vcf'}->[0] ne '1';

    push @$rows, {
      fave    => $sp->{'favourite'} ? 'Y' : '',
      species => $sp_link,
      dna     => sprintf('FASTA', $title{'dna'},  $ftp_base, $sp_dir),
      cdna    => sprintf('FASTA',  $title{'cdna'}, $ftp_base, $sp_dir),
      cds	    => sprintf('FASTA',   $title{'cds'}, $ftp_base, $sp_dir),
      ncrna   => sprintf('FASTA', $title{'rna'},  $ftp_base, $sp_dir),
      protseq => sprintf('FASTA',   $title{'prot'}, $ftp_base, $sp_dir),
      embl    => sprintf('EMBL',         $title{'embl'},  $ftp_base, $sp_dir),
      genbank => sprintf('GenBank',   $title{'genbank'}, $ftp_base, $sp_dir),
      genes   => sprintf('GTF GFF3', $title{'gtf'}, $ftp_base, $sp_dir, $title{'gff3'}, $ftp_base, $sp_dir),
      xrefs   => sprintf('TSV RDF JSON', $title{'tsv'}, $ftp_base, $sp_dir, $title{'rdf'}, $ftp_base, $sp_dir, $title{'json'}, $ftp_base, $sp_dir),
      mysql   => sprintf('MySQL',          $title{'mysql'},  $ftp_base),
      var2    => $has_vcf ? sprintf('GVF', $title{'gvf'}, $ftp_base, $sp_dir) : '-',
      var4    => $has_vcf ? sprintf('VCF', $title{'vcf'}, $ftp_base, $sp_dir) : '-',
      var3    => sprintf('VEP',    $title{'vep'}, $ftp_base),
      funcgen => $required_lookup->{'funcgen'}{$sp_dir} ? sprintf('Regulation (GFF)',      $title{'funcgen'}, $ftp_base, $sp_dir) : '-',
      bam     => $databases->{'DATABASE_RNASEQ'}        ? sprintf('BAM/BigWig',      $title{'bam'},    $ftp_base, $sp_dir) : '-',
      files   => $required_lookup->{'files'}{$sp_dir}   ? sprintf('Regulation data files', $title{'files'}, $ftp_base, $sp_dir) : '-',
    };

  }

  my $main_table           = EnsEMBL::Web::Document::Table->new($columns, $rows, { data_table => 1, exportable => 0 });
  $main_table->code        = 'FTPtable::'.scalar(@$rows);
  $main_table->{'options'}{'data_table_config'} = {iDisplayLength => 10};

  my $multi_table = $self->multi_table($rel, $version, %title);

  my $fave_text = '';
  if (scalar @$all_species > 1) {
    $fave_text = $hub->user ? 'Your favourite species are listed first.'
                  : 'Popular species are listed first. You can customise this list via our home page.';
  }

  if ($multi_table) {
    $html .= sprintf('
Multi-species data
%s
Single species data
', $multi_table->render);
  }

  $html .= sprintf(qq{

      <input type="hidden" class="panel_type" value="Content">

%s

      %s

    %s
    %s
  }, $fave_text, $main_table->render, $self->metadata, $self->add_footnotes);

  return $html;
}

sub get_misc_params {
  my ($self, $ftp, $division) = @_;
  my $species_defs = $self->hub->species_defs;
  my $version;

  if ($division) {
    $version = $species_defs->SITE_RELEASE_VERSION;
  }
  else {
    $version = $species_defs->ORIGINAL_VERSION || $species_defs->ENSEMBL_VERSION;
  }
  my $rel = "release-$version"; # Always set to use the release number rather than current to get around the delay in FTP site links updating

  my $ftp_base = $ftp;
  $ftp_base .= "/$rel";
  $ftp_base .= "/$division" if $division;

  return ($ftp_base, $version, $rel);
}

# Lookup for the types we need for species
sub required_types_for_species {
  my $self = shift;
  my %required_lookup;

  # Regulatory build
  $required_lookup{'funcgen'} = { map { $_ => 1 } qw(
    homo_sapiens mus_musculus
  )};

  # Funcgen files
  $required_lookup{'files'} = { map { $_ => 1 } qw(
    homo_sapiens mus_musculus
  )};

  return \%required_lookup;
}

sub multi_table {
  my ($self, $rel, $version, %title) = @_;
  my $hub = $self->hub;
  my $sd = $hub->species_defs;

  my $ftp_base = $sd->ENSEMBL_FTP_URL;
  unless ($ftp_base =~ /rapid/) {
    $ftp_base .= "/$rel";
  }

  my $multi_table;

  unless ($sd->NO_COMPARA) {
    $multi_table = EnsEMBL::Web::Document::Table->new([
      { key => 'database',  title => 'Database' },
      { key => 'mysql',     title => '', align => 'center' },
      { key => 'emf',       title => '', align => 'center' },
      { key => 'maf',       title => '', align => 'center' },
      { key => 'bed',       title => '', align => 'center' },
      { key => 'xml',       title => '', align => 'center' },
      { key => 'ancestral', title => '', align => 'center' }
    ], [{
      database  => 'Comparative genomics',
      mysql     => sprintf('MySQL', $title{'mysql'}, $ftp_base),
      emf       => sprintf('EMF', $title{'emf'}, $ftp_base),
      maf       => sprintf('MAF',$title{'maf'}, $ftp_base),
      bed       => sprintf('BED', $title{'bed'}, $ftp_base),
      xml       => sprintf('XML', $title{'xml'}, $ftp_base),
      ancestral => sprintf('Ancestral Alleles',$title{'ancestral'}, $ftp_base),
    }, {
      database  => 'BioMart',
      mysql     => sprintf('MySQL',$title{'mysql'}, $ftp_base),
      emf       => '-',
      maf       => '-',
      bed       => '-',
      xml       => '-',
      ancestral => '-',
    }, {
      database  => 'Stable ids',
      mysql     => sprintf('MySQL', $title{'mysql'}, $ftp_base, $version ),
      emf       => '-',
      maf       => '-',
      bed       => '-',
      xml       => '-',
      ancestral => '-',
    }], { cellpadding => 4, cellspacing => 2, id => 'ftp-table1' });
  }

  return $multi_table;
}

sub metadata {}

sub add_footnotes {
  my $self = shift;
  my $hub = $self->hub;
  my $sd = $hub->species_defs;

  my $html = qq(

To facilitate storage and download all databases are
GNU
Zip (gzip, *.gz) compressed.

About the data

The following types of data dumps are available on the FTP site.

FASTA

FASTA sequence databases of Ensembl gene, transcript and protein
model predictions. Since the
FASTA format does not permit sequence annotation,
these database files are mainly intended for use with local sequence
similarity search algorithms. Each directory has a README file with a
detailed description of the header line format and the file naming
conventions.

DNA

Masked
  and unmasked genome sequences associated with the assembly (contigs,
  chromosomes etc.).

The header line in an FASTA dump files containing DNA sequence
  consists of the following attributes :
  coord_system:version:name:start:end:strand
  This coordinate-system string is used in the Ensembl API to retrieve
  slices with the SliceAdaptor.

CDS

Coding sequences for Ensembl or ab
  initio predicted
  genes.

cDNA

cDNA sequences for Ensembl or ab
  initio predicted
  genes.

Peptides

Protein sequences for Ensembl or ab
  initio predicted
  genes.

RNA

Non-coding RNA gene predictions.

Annotated sequence

Flat files allow more extensive sequence annotation by means of
feature tables and contain thus the genome sequence as annotated by
the automated Ensembl
genome
annotation pipeline. Each nucleotide sequence record in a flat
file represents a 1Mb slice of the genome sequence. Flat files are
broken into chunks of 1000 sequence records for easier downloading.

EMBL

Ensembl database dumps in EMBL nucleotide
  sequence database format

GenBank

Ensembl database dumps
  in GenBank nucleotide sequence
  database format

MySQL

All Ensembl MySQL databases are available in text format as are
the SQL table definition files. These can be imported into any SQL
database for a local
installation of a mirror
site. Generally, the FTP directory tree contains one directory per
database. For more information about these databases and their
Application Programming Interfaces (or APIs) see the
API section.

GTF

Gene sets for each species. These files include annotations of
both coding and non-coding genes. This file format is
described here.

GFF3

GFF3 provides access to all annotated transcripts which make
up an Ensembl gene set. This file format is
described here.

  );

  unless ($sd->NO_COMPARA) {
    $html .= qq(

EMF flatfile dumps (comparative data)

Alignments of resequencing data are available for several species as
Ensembl Multi Format (EMF) flatfile dumps. The accompanying README
file describes the file format.

Also, the same format is used to dump whole-genome multiple alignments
as well as gene-based multiple alignments and phylogentic trees used
to infer Ensembl orthologues and paralogues. These files are available
in the ensembl_compara database which will be found in
the mysql
directory.

MAF (comparative data)

MAF files are provided for all pairwise alignments containing human
(GRCh38), and all multiple alignments.
The MAF file format is described here.

      );
  }

  unless ($sd->NO_VARIATION) {
    $html .= qq(

GVF (variation data)

GVF (Genome Variation Format) is a simple tab-delimited format derived
from GFF3 for variation positions across the genome.
There are GVF files for different types of variation data (e.g.
somatic variants, structural variants etc). For more information see
the "README" files in the GVF directory.

VCF (variation data)

VCF (Variant Call Format) is a text file format containing meta-information lines, a header
line, and then data lines each containing information about a position in the genome. This file format can also contain genotype information on samples for each position.
More details about the format and its specifications are available here.

VEP (variation data)

Compressed text files (called "cache files") used by the Variant Effect Predictor tool. More information about these files is available here.

BED format files (comparative data)

Constrained elements calculated using GERP are available in BED
format. For more information see the accompanying README file.

BED format is a simple line-based format. The first 3 mandatory columns
are:

chromosome name (may start with 'chr' for compliance with UCSC)

start position. This is a 0-based position

end position.

More information on the BED file format...

      );
  }

  if ($sd->HAS_API_DOCS) {
    $html .= qq(

Tarball

The entire Ensembl API is gzipped and concatenated into a single TAR file. This is updated daily.

      );
  }

  $html .= '
';

  return $html;
}

1;


text:
layout: landing
title: Landing
header: Buy our product
tagline: "A ridicolus landing page example"
group:
img : 1.jpg
---
{% include JB/setup %}

With the product you'll became more beautifull

This is the truth you will be more beautiful and attractive in order to be always perfect in every moment of your life.

You will demonstrate your authority

And to show other people what you are worth, taking advantage of all its benefits.

Finally, you can improve your lifestyle thanks to the product

That's right, you can not help it and your life will be full again and fun as long ago when you were a successful person.

Make the point of situation

Make the point of situation for persuade your customer to buy your great product.

Since using the product they are much prettier and friends make me the note.

I finally found my authority! Thanks product!

        From when I use it I can go for a walk without being ashamed of myself.

        Now I can finally take in hand my life and make it a successful life!

Now buy our amazing product!

Buying the product be able to express the best of yourself and your feelings.

Remember to subscribe to the list to receive all the tips to make the most out of the product!

          Buy now!

redpajama_set_name: RedPajamaGithub


text:
/**
 *
 */
package org.apereo.lap.controllers.api;

import java.util.ArrayList;

import org.apereo.lap.model.api.ModelOutputRecord;
import org.apereo.lap.model.api.ModelOutputResourceAssembler;
import org.apereo.lap.services.storage.ModelOutput;
import org.apereo.lap.services.storage.PersistentStorage;
import org.apereo.lap.services.storage.StorageFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.data.domain.Page;
import org.springframework.data.domain.PageImpl;
import org.springframework.data.domain.Pageable;
import org.springframework.data.web.PageableDefault;
import org.springframework.data.web.PagedResourcesAssembler;
import org.springframework.hateoas.PagedResources;
import org.springframework.web.bind.annotation.PathVariable;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RequestMethod;
import org.springframework.web.bind.annotation.RequestParam;
import org.springframework.web.bind.annotation.RestController;

/**
 * @author ggilbert
 *
 */
@RestController
public class ModelOutputController {

  @Autowired
  private StorageFactory storageFactory;

  @Autowired
  private ModelOutputResourceAssembler modelOutputResourceAssembler;

  @RequestMapping(method = RequestMethod.GET, produces = {"application/json"}, value="/api/output/{tenant}")
  public PagedResources<ModelOutputRecord> output(@PageableDefault(size = 100, page = 0) Pageable pageable,
      PagedResourcesAssembler<ModelOutput> assembler) {

    PersistentStorage<ModelOutput> persistentStorage = storageFactory.getPersistentStorage();
    Page<ModelOutput> output = persistentStorage.findAll(pageable);
    if (output == null) {
      output = new PageImpl<ModelOutput>(new ArrayList<ModelOutput>(), pageable, 0);
    }
    return assembler.toResource(output, modelOutputResourceAssembler);
  }

  @RequestMapping(method = RequestMethod.GET, produces = {"application/json"}, value="/api/output/{tenant}/student/{id}")
  public PagedResources<ModelOutputRecord> outputByStudent(@PageableDefault(size = 100, page = 0) Pageable pageable,
      PagedResourcesAssembler<ModelOutput> assembler, @PathVariable("id") final String id) {

    PersistentStorage<ModelOutput> persistentStorage = storageFactory.getPersistentStorage();
    Page<ModelOutput> output = persistentStorage.findByStudentId(id, pageable);
    if (output == null) {
      output = new PageImpl<ModelOutput>(new ArrayList<ModelOutput>(), pageable, 0);
    }
    return assembler.toResource(output, modelOutputResourceAssembler);
  }

  @RequestMapping(method = RequestMethod.GET, produces = {"application/json"}, value="/api/output/{tenant}/course/{id}")
  public PagedResources<ModelOutputRecord> outputByCourse(@PageableDefault(size = 100, page = 0) Pageable pageable,
      PagedResourcesAssembler<ModelOutput> assembler, @PathVariable("id") final String id, @RequestParam(required=false,value="lastRunOnly") boolean lastRunOnly) {

    PersistentStorage<ModelOutput> persistentStorage = storageFactory.getPersistentStorage();
    Page<ModelOutput> output = persistentStorage.findByCourseId(id, lastRunOnly, pageable);
    if (output == null) {
      output = new PageImpl<ModelOutput>(new ArrayList<ModelOutput>(), pageable, 0);
    }
    return assembler.toResource(output, modelOutputResourceAssembler);
  }

  @RequestMapping(method = RequestMethod.GET, produces = {"application/json"}, value="/api/output/{tenant}/course/{courseId}/student/{studentId}")
  public PagedResources<ModelOutputRecord> outputByStudentAndCourse(@PageableDefault(size = 100, page = 0) Pageable pageable,
      PagedResourcesAssembler<ModelOutput> assembler, @PathVariable("courseId") final String courseId, @PathVariable("studentId") final String studentId, @RequestParam(required=false,value="lastRunOnly") boolean lastRunOnly) {

    PersistentStorage<ModelOutput> persistentStorage = storageFactory.getPersistentStorage();
    Page<ModelOutput> output = persistentStorage.findByStudentIdAndCourseId(studentId, courseId, lastRunOnly, pageable);
    if (output == null) {
      output = new PageImpl<ModelOutput>(new ArrayList<ModelOutput>(), pageable, 0);
    }
    return assembler.toResource(output, modelOutputResourceAssembler);
  }

}


text:
using System;
using System.Collections.Generic;
using System.Linq;
using System.Web;
using System.Web.UI;
using System.Web.UI.WebControls;
using System.Data;
using System.Collections;

using HSoft.SQL;

using OboutInc.Calendar2;
using Obout.Grid;
using Obout.Grid.Design;

public partial class Pages_Leads_info : System.Web.UI.Page
{
    DataTable _dtEmails = new DataTable();
    DataTable _gridLeads = new DataTable();

    protected void Page_Load(object sender, EventArgs e)
    {

        if (!Page.IsPostBack)
        {
            HSoft.SQL.SqlServer _sql = new SqlServer(System.Configuration.ConfigurationManager.ConnectionStrings["ClientManager"].ConnectionString);
            String ssql = "SELECT Id,shortCode,Text FROM _LeadEmail WHERE isdeleted = 0";
            _dtEmails = _sql.GetTable(ssql);
            _sql.Close();
        }

        gridLeads.ID = "gridLeds";
            gridLeads.AllowAddingRecords = false;
            gridLeads.AllowFiltering = true;

            // grid scrolloIg
            gridLeads.ScrollingSettings.FixedColumnsPosition = GridFixedColumnsPositionType.Left;
            gridLeads.ScrollingSettings.NumberOfFixedColumns = 2;
            gridLeads.ScrollingSettings.ScrollWidth = new Unit(100, UnitType.Percentage);
            gridLeads.ScrollingSettings.ScrollHeight = 500;

            gridLeads.CallbackMode = true;
            // gridLeads.Serialize = true;
            gridLeads.AutoGenerateColumns = false;
            gridLeads.Width = Unit.Percentage(100);

            gridLeads.PageSize = 50;
            gridLeads.AllowPaging = true;
            gridLeads.AllowPageSizeSelection = false;

            // grid filter
            GridFilteringSettings gfs = new GridFilteringSettings();
            gfs.FilterPosition = GridFilterPosition.Top;
            gfs.FilterLinksPosition = GridElementPosition.Top;
            gfs.InitialState = GridFilterState.Visible;
            gridLeads.FilteringSettings = gfs;

            Column Edit = new Column();
            Edit.AllowEdit = true;
            Edit.AllowDelete = false;
            Edit.Width = "100";
            Edit.Wrap = true;
            gridLeads.Columns.Add(Edit);

            Column Id = new Column();
            Id.DataField = "Id";
            Id.HeaderText = "Id";
            Id.Visible = false;
            gridLeads.Columns.Add(Id);

            Column Number = new Column();
            Number.DataField = "Number";
            Number.HeaderText = "Number";
            Number.Width = "70";
            Number.AllowFilter = false;
            Number.ReadOnly = true;
            gridLeads.Columns.Add(Number);

            Column eMail = new Column();
            eMail.AllowFilter = true;
            eMail.DataField = "Email";
            eMail.HeaderText = "eMail";
            eMail.Width = "160";
            eMail.Wrap = true;
            eMail.ReadOnly = true;
            gridLeads.Columns.Add(eMail);

            Column Name = new Column();
            Name.AllowFilter = true;
            Name.DataField = "Name";
            Name.HeaderText = "Name";
            Name.Width = "150";
            Name.ReadOnly = true;
            gridLeads.Columns.Add(Name);

            SortedList<String, CheckBoxColumn> LeadMail = new SortedList<String, CheckBoxColumn>();
            foreach (DataRow dr in _dtEmails.Rows)
            {
                LeadMail.Add(dr["Id"].ToString(), new CheckBoxColumn());
                LeadMail[dr["Id"].ToString()].AllowFilter = false;
                LeadMail[dr["Id"].ToString()].AllowSorting = false;
                LeadMail[dr["Id"].ToString()].DataField = String.Format("Email_{0}", dr["shortCode"].ToString().Trim());
                LeadMail[dr["Id"].ToString()].HeaderText = dr["shortCode"].ToString().Trim();
                LeadMail[dr["Id"].ToString()].Width = "40";
                gridLeads.Columns.Add(LeadMail[dr["Id"].ToString()]);
            }

            Column entryDate = new Column();
            entryDate.AllowFilter = false;
            entryDate.DataField = "OrigEntryDate";
            entryDate.HeaderText = "Entry Date";
            entryDate.Width = "90";
            entryDate.DataFormatString = "{0:MM/dd/yyyy}";
            entryDate.ReadOnly = true;
            gridLeads.Columns.Add(entryDate);

            Column callLaterDate = new Column();
            callLaterDate.AllowFilter = true;
            callLaterDate.DataField = "CallLaterDate";
            callLaterDate.HeaderText = "Call Later";
            callLaterDate.Width = "140";
            callLaterDate.NullDisplayText = "missing";
            callLaterDate.DataFormatString = "{0:MM/dd/yyyy}";
            callLaterDate.ReadOnly = false;

            FilterOptionTemplateSettings fots = new FilterOptionTemplateSettings();
            fots.FilterTemplateId = "callLaterDateBetweenFilter";
            fots.FilterControlsIds = "StartDate_callLaterDate,EndDate_callLaterDate";
            fots.FilterControlsPropertyNames = "value,value";

            CustomFilterOption cfo = new CustomFilterOption();
            cfo.IsDefault = true;
            cfo.ID = "Between_callLaterDate";
            cfo.Text = "Between";
            cfo.TemplateSettings = fots;

            FilterCriteria fc = new FilterCriteria();
            fc.Values = new Hashtable();
            fc.Values["StartDate_callLaterDate"] = "1/1/1994";
            fc.Values["EndDate_callLaterDate"] = "10/1/1994";
            fc.Option = cfo;

            callLaterDate.FilterOptions.Add(cfo);
            callLaterDate.FilterCriteria = fc;

            gridLeads.Columns.Add(callLaterDate);

            Column empName = new Column();
            empName.AllowFilter = true;
            empName.DataField = "empName";
            empName.HeaderText = "Assigned To";
            empName.Width = "150";
            empName.ReadOnly = true;
            gridLeads.Columns.Add(empName);

            Column cNote = new Column();
            cNote.AllowFilter = true;
            cNote.AllowSorting = false;
            cNote.DataField = "cNote";
            cNote.HeaderText = "Comment";
            cNote.Width = "300";
            cNote.ReadOnly = true;
            cNote.Wrap = true;
            gridLeads.Columns.Add(cNote);

            Column eNote = new Column();
            eNote.AllowFilter = true;
            eNote.AllowSorting = false;
            eNote.DataField = "eNote";
            eNote.HeaderText = "Notes";
            eNote.Width = "300";
            eNote.ReadOnly = true;
            eNote.Wrap = true;
            gridLeads.Columns.Add(eNote);

//            if (!Page.IsPostBack)
//            {
                CreateLeadsGrid();
//            }

    }
    public void CreateLeadsGrid()
    {
        HSoft.SQL.SqlServer _sql = new SqlServer(System.Configuration.ConfigurationManager.ConnectionStrings["ClientManager"].ConnectionString);
        String ssql = String.Format("SELECT TOP {0} c.Id, " +
                                "       c.Number, " +
                                "       LTRIM(RTRIM(c.FirstName)) + ' ' + LTRIM(RTRIM(c.LastName)) Name, " +
                                "       c.OrigEntryDate OrigEntryDate, " +
                                "       CONVERT(nvarchar(10),l.CallLaterDate,103) CallLaterDate, " +
                                "       e.Email Email, " +
                                "       LTRIM(RTRIM(emp.FirstName)) + ' ' + LTRIM(RTRIM(emp.LastName)) empName, ", 50);
        foreach (DataRow dr in _dtEmails.Rows)
        {
            ssql = String.Format("{0} (SELECT COUNT(*) FROM LeadEmail e WHERE c.Id = e.CustomerId AND e.LeadEmailId = '{2}' AND e.isdeleted = 0) Email_{1} ,", ssql, dr["shortCode"], dr["Id"]);
        }
        ssql = String.Format("{0} nc.Note cNote,ne.Note eNote " +
                                " FROM Customer c " +
                                "   INNER JOIN Email e ON c.Id = e.CustomerId AND e.[Primary] = 1 AND e.isdeleted = 0 " +
                                "   INNER JOIN Lead l ON c.Id = l.CustomerId AND e.isdeleted = 0 " +
                                "   INNER JOIN Employee emp ON emp.Id = l.AssignedToId  AND emp.isdeleted = 0 " +
                                "   INNER JOIN PhoneNumber p ON c.Id = p.CustomerId AND p.isdeleted = 0 AND IsLead = 1 " +
                                "   INNER JOIN Note nc ON c.Id = nc.CustomerId AND nc.NoteTypeId = (SELECT Id FROM _NoteType WHERE isCustomer=1 AND isEmployee = 0) " +
                                "   INNER JOIN Note ne ON c.Id = ne.CustomerId AND ne.NoteTypeId = (SELECT Id FROM _NoteType WHERE isCustomer=1 AND isEmployee = 1 AND isPrivate=0) " +
                                " WHERE e.isdeleted = 0 " +
                                " ORDER BY OrigEntryDate DESC, Id " +
                                "", ssql);
        _gridLeads = _sql.GetTable(ssql);

        gridLeads.DataSource = _gridLeads;
        gridLeads.DataBind();

        _sql.Close();

    }

}


text:
package br.com.objectos.jabuticava.serasa.relato.factoring;

/**
 * @author marcio.endo@objectos.com.br (Marcio Endo)
 */
class ProcessamentoFake {

  public static final Processamento PROCESSAMENTO_02 = Processamento.builder()
      .numero("02")
      .mensagem("TODA   REMESSA   DESPREZADA   EM:                 12.06.2015")
      .build();

  public static final Processamento REMESSA_OK = Processamento.builder()
      .numero("01")
      .mensagem("3 - remessa total processada ok em dd/mm/aaaa")
      .build();

  private ProcessamentoFake() {
  }

}


text:
---
layout: post
title: "grep your way to freedom"
date: 2018-01-12
permalink: grep-your-way-to-freedom
categories: computers
---

I was recently eating lunch with someone who asked a question that I became somewhat obsessed with: what happens when you grep a file and append the contents to the same file? It's one of these nasty cases of modifying a file that you're reading from.

Huge thank you to [Mehmet Emre](https://cs.ucsb.edu/~emre/) & [Antal Spector-Zabusky](https://github.com/antalsz) for pair programming down this rabbit hole!

## Problem Spec
To be precise, what happens when you run the following code snippet?

```bash
# Make a junk file that just has the char a in it
echo "a" > test.txt

# Grep for a & append back to the same file
grep "a" test.txt >> test.txt
```
Okay before you run it, what are some reasonable things that *could* happen? And which of those do you think actually happens?

This nice photo (of the magical [Zion Narrows!](https://www.nps.gov/zion/planyourvisit/thenarrows.htm)) is here to avoid giving it away in case you want to think about it... scroll on for some possible answers.

![](/images/narrows.JPG)

## Possible Behavior

Okay, here are some reasonable things that could happen:

1. `grep` could finish scanning the file, then append. If this is the case, the grep would terminate, and if we ran `cat test.txt` we'd see that the test file has two a's.

2. `grep` could read from the file, find an 'a', and then append that 'a' to the file. It could then check if there was any more file to read, and see the 'a' it just wrote. If this is the case, grep would get stuck in an infinite loop, and if we killed it we would see that the test file has many a's in it.

3. This could be an error. If this is the case, grep could do nothing and return an error code.

4. Some exciting race condition that causes either a non-deterministic mix of 1 & 2, or maybe a deadlock.

## Okay so what actually happens?

... it depends!

First thing it depends upon is your OS. If you run the GNU grep (ie, on a Linux box) you get this nice error message:

```bash
grep: input file 'test.txt' is also the output
```

I was actually not expecting any implementation to do error handling like that. That's nice!

If you run the BSD `grep` (ie, on a Mac) you get:

```
$ echo "a" > test.txt
$ grep "a" test.txt >> test.txt
$ cat test.txt
a
a
```

So it's the case where it just terminated. Okay, well that's boring, so let's rinse and repeat by running the same command, and then printing out how many lines the file has:

![](/images/fixed_grep0.jpg)

And about 15 up-arrow+enters later:

![](/images/fixed_grep1.jpg)

We get stuck in an infinite loop!

How fun! We'll look into why this is happening in a second, but first,

## Follow up question

What happens if you try the same kind of command but with `cat` instead of `grep`?

```bash
# Make another junk file that just has the char a in it
echo "a" > cattest.txt

# Cat & append back to the same file
cat cattest.txt >> cattest.txt
```

## Follow up answer

We get stuck in an infinite loop right off the bat. Inconsistent behavior! Excellent.

Okay, let's look into what's actually happening. As you might guess, `cat` is simpler than `grep`, so let's figure out what's going on there first.

Since I'm running a mac, I'll be referring to Apple's recently opensourced [code for `cat`](https://opensource.apple.com/source/text_cmds/text_cmds-87/cat/cat.c) which unfortunately doesn't have line numbers- I'll copy the code we care about below but if you want to follow along in the file, you can control-F for `write`.

```c
while ((nr = read(rfd, buf, bsize)) > 0)
  for (off = 0; nr; nr -= nw, off += nw)
    if ((nw = write(wfd, buf + off, (size_t)nr)) < 0)
      err(1, "stdout");
```

So we've got a while-loop, where we read into a buffer. If we read any non-zero number of bytes, we go into a for-loop where we drain the buffer by doing writes. Once we've drained the buffer, we'll go back up to the while loop, which will do another read into the buffer, and, if we read anything we'll keep going.

So just to be really pedantic here, the reason `cat` gets stuck is that, according to [the man page for `write`](https://linux.die.net/man/2/write)

> POSIX requires that a read(2) which can be proved to occur after a write() has returned returns the new data.

which means that when we do the `write` in that for-loop, we're guaranteed that the `read` which happens after in the while-loop must see the result of that `write`. So: we read an "a", write an "a", and when we get back to the while-loop we're guaranteed that the next read will see that "a" we just wrote.

Which is why we get stuck in an infinite loop off the bat.

## Back to grep

Okay so let's go back to `grep` but before we look at the source, let me point out that there's a flag that changes the behavior:

```bash
grep --line-buffered "a" test.txt >> test.txt
```

If we run grep with the line-buffered flag then we get the same behavior as `cat` - it goes into an infinite loop even for very small files. So clearly the answer has something to do with buffering.

## Line buffering

So if we check the [grep source](https://opensource.apple.com/source/text_cmds/text_cmds-99/grep/grep.c.auto.html) for something about line buffering, we'll find that the `--line-buffered` flag calls something called `setlinebuf`

```c
	if (lbflag)
		setlinebuf(stdout);
```

[which](https://linux.die.net/man/3/setlinebuf) sets stdout to have this behavior:

> line buffered characters are saved up until a newline is output or input is read from any stream attached to a terminal device (typically stdin).

Cool. So if it spins with the `--line-buffered` flag, then probably what's causing that is that the *write buffer* is being flushed rather than the read buffer. To verify this, let's snoop on the read & write calls. On mac I'll call

```bash
sudo rwsnoop -n grep
```

to start watching for read/write calls; this will give me info about any read/write calls that grep performs. On Linux you can use `strace`. (Sidenote: you'll need to [let `dtrace` through SIP](https://apple.stackexchange.com/questions/208762/now-that-el-capitan-is-rootless-is-there-any-way-to-get-dtrace-working) to get this to work)

Okay, so after running that in one shell, I'll pop open another shell and repeat this whole grep experiment from the beginning.

![](/images/rwsnoop1.jpg)

Sweet so we can see that we've got two read calls from `grep`, the first of which reads 2 bytes, the second of which reads 0 bytes. Then we've got a write of 2 bytes. That seems pretty consistent with the behavior we're seeing. Running a few more rounds:

![](/images/rwsnoop2.jpg)

Basically the same story; we can see that now we're reading 4 bytes, and 8 bytes- again exactly what we'd expect. Skipping forward to where we go into the infinite loop:

![](/images/rwsnoop3.jpg)

The last round before we get stuck we can see that we're doing **two** reads of size 32KB before getting a read of size 0 and doing the write. This means the looping we're seeing is *definitely* not caused by buffering on the read buffer!

When we do start looping, we do three reads of size 32KB and then a 64KB write; then repeat! Okay, so now all I want to know is where are these buffer sizes coming from?

## grep source

First up: what's `grep` doing? Well if we look [at the `main`](https://opensource.apple.com/source/text_cmds/text_cmds-99/grep/grep.c.auto.html) it just processes a ton of flags, and then (very near the bottom) calls `procfile`.

`procfile` is in [util.c](https://opensource.apple.com/source/text_cmds/text_cmds-99/grep/util.c.auto.html) along with just about everything else we're going to care about. `procfile` does two things we care about, first it calls `grep_open` (which will read the file) and then `procline` (which will write the results).

```c
/*
 * Opens a file and processes it.  Each file is processed line-by-line
 * passing the lines to procline().
 */
int
procfile(const char *fn)
{ ...
  f = grep_open(fn);
  ...
  /* Process the file line-by-line */
		if ((t = procline(&ln, f->binary)) == 0 && Bflag > 0) {
      ...
```

#### Where grep reads

`grep_open` is in [file.c](https://opensource.apple.com/source/text_cmds/text_cmds-99/grep/file.c.auto.html). `grep_open` calls `grep_refill`:

```c
/*
 * Opens a file for processing.
 */
struct file *
grep_open(const char *path)
{
...
	/* Fill read buffer, also catches errors early */
	if (bufrem == 0 && grep_refill(f) != 0)

```

`grep_refill` does a read into a buffer:

```c
grep_refill(struct file *f)
{ ...
  nr = read(f->fd, buffer, MAXBUFSIZ);
```

`MAXBUFSIZ` is defined at the top of this file:

```c
#define	MAXBUFSIZ	(32 * 1024)
```

and is the exact size of the reads we were seeing. Hooray!

To recap, here's the control flow:

`main -> procfile -> grep_open -> grep_refill -> read(..., MAXBUFSIZ)`

#### Where grep writes

From before, recall that `procfile`, called `procline`. `procline` eventually calls `printline`:

```c
/*
 * Processes a line comparing it with the specified patterns.  Each pattern
 * is looped to be compared along with the full string, saving each and every
 * match, which is necessary to colorize the output and to count the
 * matches.  The matching lines are passed to printline() to display the
 * appropriate output.
 */
static int
procline(struct str *l, int nottext)
{
  ...
  printline(l, ':', matches, m);
```
which, if you scroll all the way down, on the penulitimate line, calls `fwrite`:

```c
/*
 * Prints a matching line according to the command line options.
 */
void
printline(struct str *line, int sep, regmatch_t *matches, int m)
{
  ...
  fwrite(line->dat, line->len, 1, stdout);
  putchar('\n');
```

So where `cat` was doing an honest-to-god `write`, grep goes through `fwrite`. So whatever buffer size we're using is somehow tied up in the `fwrite` source.

To summarize, here's the control flow:

`main -> procfile -> procline -> printline -> fwrite`

## fwrite buffer size

Figuring out where `fwrite` sets it's buffer size was more challenging- props here to Mehmet for his spelunking skills. Here I'll show you the relevant control flow up front:

`fwrite --> __sfvwrite --> prep_write --> __swsetup --> smakebuf --> __swhatbuf`

I won't walk you through all those layers of control flow but you can definitely check it out yourself if you want; starting at [`fwrite`](https://opensource.apple.com/source/Libc/Libc-167/stdio.subproj/fwrite.c.auto.html) and ending at [`__swhatbuf`](https://opensource.apple.com/source/Libc/Libc-1244.1.7/stdio/FreeBSD/makebuf.c.auto.html).

**Interesting Side-note!** The implementation of makebuf.c in the latest apple OS (High Sierra) is *rather different* than in the implementation in [an older version](https://opensource.apple.com/source/Libc/Libc-167/stdio.subproj/makebuf.c.auto.html). Perhaps because Apple redid their file system for High Sierra? I know this because I went through an existential crisis (is everything I know a lie?) trying to figure out why the behavior I was seeing was inconsistent with the source code before I figured out it was the wrong version!

Okay, anyways, let's take a look at the `__swhatbuf` that's in High Sierra, because that's what I'm running:

```c
#ifdef FEATURE_SMALL_STDIOBUF
# define MAXBUFSIZE	(1 << 12)
#else
# define MAXBUFSIZE	(1 << 16)
#endif

#define TTYBUFSIZE	4096

...

/*
 * Internal routine to determine `proper' buffering for a file.
 */
int
__swhatbuf(fp, bufsize, couldbetty)
	FILE *fp;
	size_t *bufsize;
	int *couldbetty;
{
	struct stat st;

	if (fp->_file < 0 || _fstat(fp->_file, &st) < 0) {
		*couldbetty = 0;
		*bufsize = BUFSIZ;
		return (__SNPT);
	}

	/* could be a tty iff it is a character device */
	*couldbetty = (st.st_mode & S_IFMT) == S_IFCHR;
	if (st.st_blksize <= 0) {
		*bufsize = BUFSIZ;
		return (__SNPT);
	}

	/*
	 * Optimise fseek() only if it is a regular file.  (The test for
	 * \_\_sseek is mainly paranoia.)  It is safe to set _blksize
	 * unconditionally; it will only be used if __SOPT is also set.
	 */
	fp->_blksize = *bufsize = st.st_blksize > MAXBUFSIZE ? MAXBUFSIZE : st.st_blksize;
	return ((st.st_mode & S_IFMT) == S_IFREG && fp->_seek == __sseek ?
	    __SOPT : __SNPT);
}

```

What I think is going on here is that we're setting \*bufsize which is being "returned" to the caller through the pointer. The first two "ifs" don't apply to us- we're not dealing with an empty file, and we're not a character device. So in the final block of code, we set \*bufsize to 2 different values depending on whether `st.st_blksize > MAXBUFSIZE`, either to

`MAXBUFSIZE`

or to

`st.st_blksize`

And actually, the top of the file says `MAXBUFSIZE` is `1 << 16` (which is 64KB) so we're golden. For fun we can check that the st_blksize is larger than 64KB to be sure that's the branch we're taking. A quick c script that looks a lot like `__swhatbuf` will do the trick:

```c
#include <sys/types.h>
#include <sys/stat.h>
#include <unistd.h>
#include <stdio.h>

int main()
{
    struct stat st;
    FILE *fp;
    fp = fopen("test.txt", "r");
    fstat(fp->_file, &st);
    printf("st_blksize was: %i\n", st.st_blksize);
    return 0;
 }
```

Compiling and running it:

```bash
$ gcc deleteme.c & ./a.out
st_blksize was: 4194304
```

Sweet, so st_blksize is 4MB which is definitely bigger than 64KB. So that's why that's the transition point between terminating & going into an infinite loop! 🎉

## Conclusions

1) Weird things happen when you read from files you write to.

2) The behaviors we saw had to do with buffering of the write buffer.

3) It's rad that you can read open-source code to figure out exactly what's going on. Grep your way to freedom!

## Appendix

I was looking at both the freebsd & apple's open source implementations of `fwrite` and they were almost identical... except that freebsd's version has both an integer overflow check & a lock. It's a little alarming that apple's version is missing those...it'd be interesting to see if you can get some weird behavior to trigger.

![](/images/mildly_alarming.jpg)

-------

Thanks also to [Dan Luu](https://danluu.com/) & [Vaibhav Sagar](http://vaibhavsagar.com/) for their feedback on a draft, and to [Tobin Yehle](http://tobin.yehle.io/pottery) for both proofreading a draft and figuring out I was looking at the wrong version of the makebuf.c file!


text:
include_recipe "apt" # this runs apt-get update which gets the latest nagios-nrpe-server package

include_recipe "nrpe"

template "#{node['nrpe']['conf_dir']}/nrpe.d/my_custom_nrpe.cfg" do
  source "my_custom_nrpe.cfg.erb"
  owner node['nrpe']['user']
  group node['nrpe']['group']
  mode 00644
  notifies :restart, "service[#{node['nrpe']['service_name']}]"
end

cookbook_file "/usr/lib/nagios/plugins/check_nfs.sh" do
  source "plugins/check_nfs.sh"
  action :create_if_missing
  mode	0755
  only_if { platform?("debian", "ubuntu")  }
end

service node['nrpe']['service_name'] do
  action [:start, :enable]
  supports :restart => true, :reload => true, :status => true
end


text:
'''
the standard interface for auto init db schemas of different types
'''

class InitDBModel(object):

    @classmethod
    def init_schema(cls):
        raise RuntimeError('not implemented operations')


text:
package org.elasticsearch.gradle;

import java.io.IOException;
import java.io.InputStream;
import java.util.HashMap;
import java.util.Map;
import java.util.Properties;

/**
 * Accessor for shared dependency versions used by elasticsearch, namely the elasticsearch and lucene versions.
 */
public class VersionProperties {

    public static String getElasticsearch() {
        return elasticsearch;
    }

    public static Version getElasticsearchVersion() {
        return Version.fromString(elasticsearch);
    }

    public static String getLucene() {
        return lucene;
    }

    public static String getBundledJdk(final String platform) {
        switch (platform) {
            case "darwin": // fall trough
            case "mac":
                return bundledJdkDarwin;
            case "linux":
                return bundledJdkLinux;
            case "windows":
                return bundledJdkWindows;
            default:
                throw new IllegalArgumentException("unknown platform [" + platform + "]");
        }
    }

    public static String getBundledJdkVendor() {
        return bundledJdkVendor;
    }

    public static Map<String, String> getVersions() {
        return versions;
    }

    private static final String elasticsearch;
    private static final String lucene;
    private static final String bundledJdkDarwin;
    private static final String bundledJdkLinux;
    private static final String bundledJdkWindows;
    private static final String bundledJdkVendor;
    private static final Map<String, String> versions = new HashMap<String, String>();

    static {
        Properties props = getVersionProperties();
        elasticsearch = props.getProperty("elasticsearch");
        lucene = props.getProperty("lucene");
        bundledJdkVendor = props.getProperty("bundled_jdk_vendor");
        final String bundledJdk = props.getProperty("bundled_jdk");
        bundledJdkDarwin = props.getProperty("bundled_jdk_darwin", bundledJdk);
        bundledJdkLinux = props.getProperty("bundled_jdk_linux", bundledJdk);
        bundledJdkWindows = props.getProperty("bundled_jdk_windows", bundledJdk);

        for (String property : props.stringPropertyNames()) {
            versions.put(property, props.getProperty(property));
        }
    }

    private static Properties getVersionProperties() {
        Properties props = new Properties();
        InputStream propsStream = VersionProperties.class.getResourceAsStream("/version.properties");
        if (propsStream == null) {
            throw new IllegalStateException("/version.properties resource missing");
        }
        try {
            props.load(propsStream);
        } catch (IOException e) {
            throw new IllegalStateException("Failed to load version properties", e);
        }
        return props;
    }

    public static boolean isElasticsearchSnapshot() {
        return elasticsearch.endsWith("-SNAPSHOT");
    }
}


text:
void opa_runtime_error(const char *loc, int row, int col, const char *msg);

#endif


text:
import os
import glob
import numpy as np
import warnings
from skimage import measure, morphology
from scipy import ndimage
from astropy.table import Table, Column
import astropy.units as u
from astropy.coordinates import SkyCoord
from astropy.stats import gaussian_fwhm_to_sigma, sigma_clipped_stats
from astropy.convolution import convolve, convolve_fft, Gaussian2DKernel
from astroquery.vizier import Vizier

# For debugging
import matplotlib.pyplot as plt

# Add the AstroImage class
import astroimage as ai

# Add the header handler to the BaseImage class
from Mimir_header_handler import Mimir_header_handler
ai.set_instrument('2MASS')

# This script will read in the background level estimated for each on-target
# image in the previous step. The background level in dimmest parts of the
# on-target image will be directly computed, and the residual between the direct
# estimate and the interpolation will be stored. The distribution of these
# residual will be used to estimate which interpolated background levels can be
# trusted.

#==============================================================================
# *********************** CUSTOM USER CODE ************************************
# this is where the user specifies where the raw data is stored
# and some of the subdirectory structure to find the actual .FITS images
#==============================================================================

# This is the location of all PPOL reduction directory
PPOL_dir = 'C:\\Users\\Jordan\\FITS_data\\Mimir_data\\PPOL_Reduced\\201611'

# Build the path to the S3_Asotremtry files
S3_dir = os.path.join(PPOL_dir, 'S3_Astrometry')

# This is the location where all pyPol data will be saved
pyPol_data = 'C:\\Users\\Jordan\\FITS_data\\Mimir_data\\pyPol_Reduced\\201611'

# This is the directory where the 2MASS tiles of the targets have been saved
# Go to "http://hachi.ipac.caltech.edu/" to download 2MASS tiles
TMASSdir  = ".\\2MASSimages"

# Setup new directory for masks
maskDir = os.path.join(pyPol_data, 'Masks')
if (not os.path.isdir(maskDir)):
    os.mkdir(maskDir, 0o755)

starMaskDir = os.path.join(maskDir, 'starMasks')
if (not os.path.isdir(starMaskDir)):
    os.mkdir(starMaskDir, 0o755)

# Read in the indexFile data and select the filenames
indexFile = os.path.join(pyPol_data, 'reducedFileIndex.csv')
fileIndex = Table.read(indexFile, format='csv')
useRows   = np.where(fileIndex['USE'])
fileIndex = fileIndex[useRows]

################################################################################
def find_2MASS_flux(array):
    # Identify which pixels have acceptable "background" levels. Start by
    # grabbing the image statistics
    mean, median, stddev = sigma_clipped_stats(array)

    # Idesntify pixels more than 2-sigma above the background
    fgdThresh = median + 2.0*stddev
    fgdRegion = array > fgdThresh

    # Repeat the classification withiout the *definitely* nebular pixels
    bkgPix = np.logical_not(fgdRegion)
    mean, median, stddev = sigma_clipped_stats(array[bkgPix])
    fgdThresh = median + 2.0*stddev
    fgdRegion = array > fgdThresh

    # Clean the foreground ID region
    all_labels  = measure.label(fgdRegion)
    all_labels1 = morphology.remove_small_objects(all_labels, min_size=50)
    fgdRegion   = all_labels1 > 0

    # Grab the part *only* connected to the central, nebular region
    ny, nx = fgdRegion.shape
    all_labels   = measure.label(fgdRegion)
    nebularLabel = all_labels[ny//2, nx//2]
    nebularMask  = all_labels == nebularLabel
    starMask     = np.logical_and(
        all_labels > 0,
        all_labels != nebularLabel
    )
    all_labels  = measure.label(starMask)
    all_labels1 = morphology.remove_small_objects(all_labels, min_size=50)
    starMask    = all_labels1 > 0

    # Dilate a TOOON to be conservatine...
    nebularSigma = 20.0 * gaussian_fwhm_to_sigma    # FWHM = 3.0

    # Build a kernel for detecting pixels above the threshold
    nebularKernel = Gaussian2DKernel(nebularSigma, x_size=41, y_size=41)
    nebularKernel.normalize()
    nebularMask = convolve_fft(
        nebularMask.astype(float),
        nebularKernel.array
    )
    nebularMask = (nebularMask > 0.01)

    # Expand a second time to be conservative
    nebularMask = convolve_fft(
        nebularMask.astype(float),
        nebularKernel.array
    )
    nebularMask = (nebularMask > 0.01)

    # Do a less aggressive dilation of the stellar mask
    stellarSigma = 10.0 * gaussian_fwhm_to_sigma    # FWHM = 3.0

    # Build a kernel for detecting pixels above the threshold
    stellarKernel = Gaussian2DKernel(stellarSigma, x_size=41, y_size=41)
    stellarKernel.normalize()
    stellarMask = convolve_fft(
        fgdRegion.astype(float),
        stellarKernel.array
    )
    stellarMask = (stellarMask > 0.01)

    # Recombine the nebular and stellar components
    fgdRegion = np.logical_or(nebularMask, stellarMask)

    # Return the flux-bright pixels to the user
    return fgdRegion

################################################################################

# Read in the Kokopelli Mask
kokopelliMask = ai.reduced.ReducedScience.read('kokopelliMask.fits')

# Dilate the mask in order to be more conservative.
kokopelliMask.data = ndimage.binary_dilation(kokopelliMask.data, iterations=8).astype(int)

# Construct the 2MASS masks and save to disk
# Read in all the 2MASS images and store them in a dictionary for quick reference
TMASS_Hfiles = np.array(glob.glob(os.path.join(TMASSdir, '*H.fits')))
TMASS_Kfiles = np.array(glob.glob(os.path.join(TMASSdir, '*Ks.fits')))

# Read in the 2MASS images
TMASS_HimgList = [ai.reduced.ReducedScience.read(f) for f in TMASS_Hfiles]
TMASS_KimgList = [ai.reduced.ReducedScience.read(f) for f in TMASS_Kfiles]

# Convert the images to "background masks"
for img in TMASS_HimgList:
    # Construct the output name for this mask
    base        = os.path.basename(img.filename)
    targetMask  = base.split('.')[0] + '_mask.fits'
    outFilename = os.path.join(TMASSdir, targetMask)

    # Skip files that have already been done
    if os.path.isfile(outFilename): continue

    print('Building background mask for {}'.format(os.path.basename(img.filename)))
    tmp = img.copy()
    tmp.data = find_2MASS_flux(img.data).astype(int)
    tmp.write(outFilename, dtype=np.uint8)

for img in TMASS_KimgList:
    # Construct the output name for this mask
    base        = os.path.basename(img.filename)
    targetMask  = base.split('.')[0] + '_mask.fits'
    outFilename = os.path.join(TMASSdir, targetMask)

    # Skip files that have already been done
    if os.path.isfile(outFilename): continue

    print('Building background mask for {}'.format(os.path.basename(img.filename)))
    tmp = img.copy()
    tmp.data = find_2MASS_flux(img.data).astype(int)
    tmp.write(outFilename, dtype=np.uint8)

# Now that the 2MASS files have been read in, it is safe to set the Mimir_header_handler
ai.set_instrument('Mimir')
ai.reduced.ReducedScience.set_header_handler(Mimir_header_handler)

# Set the Vizier download size to be unlimited
Vizier.ROW_LIMIT = -1

# Group by target HWP
groupedFileIndex = fileIndex.group_by(['GROUP_ID', 'HWP'])

#Loop through each file in the fileList variable
numberOfFiles = len(fileIndex)
bkgLevels = fileIndex['BACKGROUND']
print('{0:3.1%} complete'.format(0), end='\r')

iRow = 0
for group in groupedFileIndex.groups:
    # Increment the row counter
    iRow += len(group)

    # Grab the relevant information for this group
    thisTarget     = np.unique(group['TARGET'])[0]
    thisFilter     = np.unique(group['FILTER'])[0]

    # Re-group by dither pointing
    ABBAsubGroups = group.group_by(['AB'])

    for ABBAgroup in ABBAsubGroups.groups:
        # Grab the relevant information for this subgroup
        thisAB = np.unique(ABBAgroup['AB'])[0]

        # If this is an on-target (A) subgroup, then skip it!
        if thisAB == 'A': continue

        # Grab the off-target files
        Bfiles = []
        maskFilenames = []
        for thisFile in ABBAgroup['FILENAME']:
            # Append the B-file to use
            Bfiles.append(os.path.join(S3_dir, thisFile))

            # BUild the mask name
            maskBasename = os.path.basename(thisFile)
            maskFilenames.append(os.path.join(starMaskDir, maskBasename))

    # Check if the file has already been written and skip those which have been
    if all([os.path.isfile(f) for f in maskFilenames]): continue

    # Read in the off-target frames
    Bimgs = [ai.reduced.ReducedScience.read(f) for f in Bfiles]

    numBimgs = len(Bimgs)
    if numBimgs > 1:
        # Combine the images to get a quick map of the region to download
        BimgStack = ai.utilitywrappers.ImageStack(Bimgs, gobble=False)
        BimgStack.align_images_with_wcs()

        # Determine the boundaries of the region to download 2MASS data
        referenceImage = BimgStack.imageList[0]
    else:
        referenceImage = Bimgs[0]

    # Get the image shape and coordinates
    ny, nx = referenceImage.shape
    lfrt, bttp = referenceImage.wcs.wcs_pix2world([0, ny], [0, nx], 0)
    lf, rt = lfrt
    bt, tp = bttp

    # Grab the maximum width and the median (RA, Dec)
    RAcen, DecCen = 0.5*(lf + rt), 0.5*(bt + tp)
    height = (tp - bt)*u.deg
    width  = (lf - rt)*np.cos(np.deg2rad(DecCen))*u.deg

    with warnings.catch_warnings():
        warnings.simplefilter("ignore")

        # Download the 2MASS point source catalog
        tmassCatalog = Vizier.query_region(
            SkyCoord(
                ra=RAcen, dec=DecCen,
                unit=(u.deg, u.deg),
                frame='fk5'
            ),
            width=width,
            height=height,
            catalog='II/246/out'
        )[0]

        # Cut off low SNR detections
        tmassFilter = referenceImage.filter[0]
        tmassSNR    = tmassCatalog[tmassFilter+'snr']
        goodDetections = np.logical_and(
            tmassSNR.data.data > 5.0,
            np.logical_not(tmassSNR.mask)
        )
        goodDetections = np.logical_and(
            goodDetections,
            np.logical_not(tmassCatalog[tmassFilter+'mag'].mask)
        )

        # Cull the bad data
        tmassCatalog = tmassCatalog[goodDetections]

    # Grab the RAs, Decs, and magnitudes
    RAs, Decs = tmassCatalog['_RAJ2000'], tmassCatalog['_DEJ2000']
    mags      = tmassCatalog[tmassFilter+'mag']

    # Loop through each file and build the preliminary mask
    starMasks = []
    for thisImg in Bimgs:
        # # Read in the image and store it for possible later use
        # thisImg = ai.reduced.ReducedScience.read(Bfile)
        #
        # # Attempt to recover a background estimate. If not, then just fill with -1e6
        # # Locate the non-stellar pixels in this image
        # photAnalyzer = ai.utilitywrappers.PhotometryAnalyzer(thisImg)
        # try:
        #     _, psfParams = photAnalyzer.get_psf()
        #     FWHM = 2.355*np.sqrt(psfParams['sminor']*psfParams['smajor'])
        # except:
        #     FWHM = 4.5

        # xs, ys = thisImg.get_sources(FWHMguess = FWHM, minimumSNR = 3.5,
        #     satLimit = 1e20, edgeLimit = 21)
        # starFluxes, fluxUncerts = photAnalyzer.aperture_photometry(
        #     xs, ys, FWHM, 24, 26, mask=(thisImg.data < -1e4)
        # )
        #
        # # Catch bad stars
        # kokopelliArtifacts = kokopelliMask.data[ys.round().astype(int), xs.round().astype(int)]
        #
        # # Look through the stars in Kokopelly and determine which are *real*
        # realStars          = (kokopelliArtifacts.astype(int)*starFluxes > 4e3)
        # kokopelliArtifacts = np.logical_and(
        #     kokopelliArtifacts,
        #     np.logical_not(realStars)
        # )
        #
        # # Only keep those stars which are not kokopilli artifacts
        # goodInds    = np.where(
        #     np.logical_and(
        #         starFluxes > 0,
        #         np.logical_not(kokopelliArtifacts)
        #     )
        # )
        # xs         = xs[goodInds]
        # ys         = ys[goodInds]
        # starFluxes = starFluxes[goodInds]

        # Now simply mask *any* of the stars downloaded
        xs, ys    = thisImg.wcs.wcs_world2pix(RAs, Decs, 0)
        starRadii = 35 - 1.5*mags

        # Loop through each star and make its mask
        ny, nx   = thisImg.shape
        yy, xx   = np.mgrid[0:ny, 0:nx]
        starMask = False
        for xs1, ys1, rs in zip(xs, ys, starRadii):
            if not np.isfinite(rs): import pdb;
            # Compute the distances from this star
            # Mask any pixels within 1 radius of this star
            starMask = np.logical_or(
                starMask,
                np.sqrt((xx - xs1)**2 + (yy - ys1)**2) < rs
            )

        # Store the mask for later use
        starMasks.append(starMask)

    # # If more than one image exists in this group, then do a secondary pass to
    # # locate the dimmer stars
    # numBimgs = len(Bimgs)
    # if numBimgs == 1:
    #     # Grab the available Bimg
    #     tmpImg  = Bimgs[0].copy()
    #
    #     # Smooth the data to look for lower SNR stars
    #     tmpImg.data = ndimage.median_filter(tmpImg.data, 3)
    #
    #     # Construct a temporary image to do another pass as the low SNR stars
    #     tmpFWHM = np.sqrt(FWHM**2 + (0.5*3)**2)
    #     xs, ys = tmpImg.get_sources(FWHMguess = tmpFWHM, minimumSNR = 2.5,
    #         satLimit = 1e20, edgeLimit = 21)
    #     starFluxes, fluxUncerts = photAnalyzer.aperture_photometry(
    #         xs, ys, tmpFWHM, 24, 26, mask=(tmpImg.data < -1e4)
    #     )
    #
    #     # Catch bad stars
    #     kokopelliArtifacts = kokopelliMask.data[ys.round().astype(int), xs.round().astype(int)]
    #
    #     # Look through the stars in Kokopelly and determine which are *real*
    #     realStars          = (kokopelliArtifacts.astype(int)*starFluxes > 4e3)
    #     kokopelliArtifacts = np.logical_and(
    #         kokopelliArtifacts,
    #         np.logical_not(realStars)
    #     )
    #
    #     # Only keep those stars which are not kokopilli artifacts
    #     goodInds    = np.where(
    #         np.logical_and(
    #             starFluxes > 0,
    #             np.logical_not(kokopelliArtifacts)
    #         )
    #     )
    #     xs         = xs[goodInds]
    #     ys         = ys[goodInds]
    #     starFluxes = starFluxes[goodInds]
    #     starRadii  = 5*np.log10(starFluxes)
    #
    #     # Loop through each star and make its mask
    #     ny, nx   = thisImg.shape
    #     yy, xx   = np.mgrid[0:ny, 0:nx]
    #     starMask = starMasks[0]
    #     for xs1, ys1, rs in zip(xs, ys, starRadii):
    #         if not np.isfinite(rs): import pdb;
    #         # Compute the distances from this star
    #         # Mask any pixels within 1 radius of this star
    #         starMask = np.logical_or(
    #             starMask,
    #             np.sqrt((xx - xs1)**2 + (yy - ys1)**2) < rs
    #         )
    #
    #     # Store the mask for later use
    #     starMasks[0] = starMask
    #
    # elif numBimgs > 1:
    #     # Loop through each
    #     for iImg in range(numBimgs):
    #         # Determine which image is the primary image and which is secondary
    #         if iImg == 0:
    #             thisImg  = Bimgs[0]
    #             otherImg = Bimgs[1]
    #         elif iImg == 1:
    #             thisImg  = Bimgs[1]
    #             otherImg = Bimgs[0]
    #         else:
    #             print('What?! How did you even get here?!')
    #             import pdb; pdb.set_trace()
    #
    #         # Grab the corresponding mask
    #         thisMask = starMasks[iImg]
    #
    #         # Subtract the two images from eachother
    #         diffData = otherImg.data - thisImg.data
    #
    #         # Smooth the difference image
    #         median9Data  = ndimage.median_filter(diffData, 9)
    #
    #         # LOOK FOR DIVOTS IN GENERAL MEDIAN FILTERED IMAGE
    #         # Locate pixels less than negative 2-sigma
    #         mean9, median9, stddev9 = sigma_clipped_stats(median9Data)
    #         starDivots = np.nan_to_num(median9Data) < (mean9 -4*stddev9)
    #
    #         # Remove anything that is smaller than 20 pixels
    #         all_labels  = measure.label(starDivots)
    #         all_labels1 = morphology.remove_small_objects(all_labels, min_size=20)
    #         label_hist, label_bins = np.histogram(
    #             all_labels1,
    #             bins=np.arange(all_labels1.max() - all_labels1.min())
    #         )
    #         label_mode  = label_bins[label_hist.argmax()]
    #         starDivots = all_labels1 != label_mode
    #
    #         # Remove any pixels along extreme top
    #         starDivots[ny-10:ny,:] = False
    #
    #         # Dialate the starDivots mask
    #         stellarSigma = 5.0 * gaussian_fwhm_to_sigma    # FWHM = 3.0
    #
    #         # Build a kernel for detecting pixels above the threshold
    #         stellarKernel = Gaussian2DKernel(stellarSigma, x_size=41, y_size=41)
    #         stellarKernel.normalize()
    #         starDivots = convolve_fft(
    #             starDivots.astype(float),
    #             stellarKernel.array
    #         )
    #         starDivots = (starDivots > 0.01)
    #
    #         # Compbine the divots mask and the original mask
    #         fullMask = np.logical_or(thisMask, starDivots)
    #
    #         # Store the mask back in its list
    #         starMasks[iImg] = ai.reduced.ReducedScience(fullMask.astype(int))
    #
    # # Do a finel loop-through to make sure there is as much agreement between
    # # the two masks as possible
    # if numBimgs > 1:
    #     # Construct an image stack and compute image offsets
    #     BimgStack = ai.utilitywrappers.ImageStack(Bimgs)
    #     dx, dy = BimgStack.get_wcs_offsets(BimgStack)
    #
    #     try:
    #         starMask0 = starMasks[0].copy()
    #         starMask1 = starMasks[1].copy()
    #     except:
    #         print('Why are there not 2 starMasks?')
    #         import pdb; pdb.set_trace()
    #
    #     for iMask in range(numBimgs):
    #         # Determine which image is the primary image and which is secondary
    #         if iMask == 0:
    #             dx1 = dx[1] - dx[0]
    #             dy1 = dy[1] - dy[0]
    #             thisMask  = starMask0
    #             otherMask = starMask1
    #         elif iMask == 1:
    #             dx1 = dx[0] - dx[1]
    #             dy1 = dy[0] - dy[1]
    #             thisMask  = starMask1
    #             otherMask = starMask0
    #         else:
    #             print('What?! How did you even get here?!')
    #             import pdb; pdb.set_trace()
    #
    #         # Shift the mask accordingly
    #         shiftedOtherMask = otherMask.shift(dx1, dy1)
    #
    #         # Combine this mask and the shifted mask
    #         fullMask = np.logical_or(
    #             thisMask.data,
    #             shiftedOtherMask.data
    #         )
    #
    #         # Store the mask for a final write-to-disk
    #         starMasks[iMask] = fullMask

    # Look through the masks and write to disk
    for maskFile, starMask in zip(maskFilenames, starMasks):
        try:
            # Write the mask to disk
            maskImg = ai.reduced.ReducedScience(starMask.astype(int))
            maskImg.write(maskFile, dtype=np.uint8)

        except:
            print('Failed to save file {}'.format(maskFile))

    # Update on progress
    print('{0:3.1%} complete'.format(iRow/numberOfFiles), end='\r')

# Alert the user that everything is complete
print('{0:3.1%} complete'.format(1), end='\n\n')

print('Done!')


text:
package org.apache.hadoop.hdfs;

import static org.junit.Assert.assertTrue;

import com.google.common.base.Supplier;
import org.apache.hadoop.crypto.key.kms.KMSClientProvider;
import org.apache.hadoop.crypto.key.kms.server.MiniKMS;
import org.apache.hadoop.security.Credentials;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.token.Token;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.test.GenericTestUtils;
import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
import org.mockito.internal.util.reflection.Whitebox;

import java.io.File;
import java.util.Arrays;
import java.util.UUID;

public class TestEncryptionZonesWithKMS extends TestEncryptionZones {

  private MiniKMS miniKMS;

  @Override
  protected String getKeyProviderURI() {
    return KMSClientProvider.SCHEME_NAME + "://" +
        miniKMS.getKMSUrl().toExternalForm().replace("://", "@");
  }

  @Before
  public void setup() throws Exception {
    File kmsDir = new File("target/test-classes/" +
        UUID.randomUUID().toString());
    Assert.assertTrue(kmsDir.mkdirs());
    MiniKMS.Builder miniKMSBuilder = new MiniKMS.Builder();
    miniKMS = miniKMSBuilder.setKmsConfDir(kmsDir).build();
    miniKMS.start();
    super.setup();
  }

  @After
  public void teardown() {
    super.teardown();
    miniKMS.stop();
  }

  @Override
  protected void setProvider() {
  }

  @Test(timeout = 120000)
  public void testCreateEZPopulatesEDEKCache() throws Exception {
    final Path zonePath = new Path("/TestEncryptionZone");
    fsWrapper.mkdir(zonePath, FsPermission.getDirDefault(), false);
    dfsAdmin.createEncryptionZone(zonePath, TEST_KEY);
    @SuppressWarnings("unchecked")
    KMSClientProvider kcp = (KMSClientProvider) Whitebox
        .getInternalState(cluster.getNamesystem().getProvider(), "extension");
    assertTrue(kcp.getEncKeyQueueSize(TEST_KEY) > 0);
  }

  @Test(timeout = 120000)
  public void testDelegationToken() throws Exception {
    final String renewer = "JobTracker";
    UserGroupInformation.createRemoteUser(renewer);

    Credentials creds = new Credentials();
    Token<?> tokens[] = fs.addDelegationTokens(renewer, creds);
    DistributedFileSystem.LOG.debug("Delegation tokens: " +
        Arrays.asList(tokens));
    Assert.assertEquals(2, tokens.length);
    Assert.assertEquals(2, creds.numberOfTokens());

    // If the dt exists, will not get again
    tokens = fs.addDelegationTokens(renewer, creds);
    Assert.assertEquals(0, tokens.length);
    Assert.assertEquals(2, creds.numberOfTokens());
  }

  @Test(timeout = 120000)
  public void testWarmupEDEKCacheOnStartup() throws Exception {
    final Path zonePath = new Path("/TestEncryptionZone");
    fsWrapper.mkdir(zonePath, FsPermission.getDirDefault(), false);
    dfsAdmin.createEncryptionZone(zonePath, TEST_KEY);

    @SuppressWarnings("unchecked")
    KMSClientProvider spy = (KMSClientProvider) Whitebox
        .getInternalState(cluster.getNamesystem().getProvider(), "extension");
    assertTrue("key queue is empty after creating encryption zone",
        spy.getEncKeyQueueSize(TEST_KEY) > 0);

    conf.setInt(
        DFSConfigKeys.DFS_NAMENODE_EDEKCACHELOADER_INITIAL_DELAY_MS_KEY, 0);
    cluster.restartNameNode(true);

    GenericTestUtils.waitFor(new Supplier<Boolean>() {
      @Override
      public Boolean get() {
        final KMSClientProvider kspy = (KMSClientProvider) Whitebox
            .getInternalState(cluster.getNamesystem().getProvider(),
                "extension");
        return kspy.getEncKeyQueueSize(TEST_KEY) > 0;
      }
    }, 1000, 60000);
  }
}


text: export const CELL_SIZE = 32;


text: "use strict";var l={"NZL":["NZL","Nya Zeeland"],"AUS":["AUS","Australien"]};(this?this:window)['DvtBaseMapManager']['_UNPROCESSED_MAPS'][2].push(["australia","countries",l]);


text:
package org.adonai.fx;

import java.io.BufferedReader;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.nio.charset.StandardCharsets;
import java.util.stream.Collectors;
import org.commonmark.node.Node;
import org.commonmark.parser.Parser;
import org.commonmark.renderer.html.HtmlRenderer;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

public class HelpController extends AbstractController {

  private Logger log = LoggerFactory.getLogger(HelpController.class);

  public void render() {
    String id = getMainController().getCenter().getId();
    if (id != null) {
      InputStream resourceAsStream = getClass().getClassLoader().getResourceAsStream("help/" + id + ".md");
      if (resourceAsStream != null) {
        Parser parser = Parser.builder().build();
        String asString = new BufferedReader(
            new InputStreamReader(resourceAsStream, StandardCharsets.UTF_8))
            .lines()
            .collect(Collectors.joining("\n"));
        Node document = parser.parse(asString);
        HtmlRenderer renderer = HtmlRenderer.builder().build();
        String html = renderer.render(document);

      }
    }
  }
}


text:
<?php  namespace Jacopo\Authentication\Tests\Unit;

/**
 * Class DbTestCase
 *
 * @author jacopo beschi jacopo@jacopobeschi.com
 */
use Artisan, DB, Closure, App, Config;
use BadMethodCallException;
use Illuminate\Support\Collection;
use Illuminate\Validation\DatabasePresenceVerifier;
use Illuminate\Validation\Factory;

class DbTestCase extends TestCase
{

    /*
	|--------------------------------------------------------------------------
	| Database connection
	|--------------------------------------------------------------------------
	| Edit the file "packages/jacopo/laravel-authentication-acl/config/testing/database.php"
    | to change the database connection for the tests.
    |
    */
    protected $connection_info;
    protected $current_connection;

    /*
	|--------------------------------------------------------------------------
	| Other
	|--------------------------------------------------------------------------
    |
    */
    // used for iterative make
    protected $times = 1;
    protected $artisan;
    protected $faker;

    public function setUp()
    {
        parent::setUp();
        $this->artisan = $this->app->make('artisan');
        $this->faker = \Faker\Factory::create();
        $this->setupDbConnection();
        $this->overwriteDatabasePresenceVerifierForTesting();
    }

    protected function setupDbConnection()
    {
        $this->getConnectionInfo();
        $this->overrideCurrentConnection();
        $this->cleanTables();
        $this->createTestDbSchema();
    }

    protected function getConnectionInfo()
    {
        $this->current_connection = Config::get('laravel-authentication-acl::database.default');
        $this->connection_info = Config::get("laravel-authentication-acl::database.connections.{$this->current_connection}");
    }

    protected function overrideCurrentConnection()
    {
        $this->app['config']->set('database.default', 'testbench');
        $this->app['config']->set('database.connections.testbench', $this->connection_info);
    }

    protected function cleanTables()
    {
        $this->callCleanMethod();
    }

    /**
     * Define environment setup.
     *
     * @param  \Illuminate\Foundation\Application $app
     * @return void
     */
    protected function getEnvironmentSetUp($app)
    {
        // reset base path to point to our package's src directory
        $app['path.base'] = __DIR__ . '/../../src';
    }

    /**
     * @return mixed
     */
    protected function callCleanMethod()
    {
        return $this->{"clean" . $this->current_connection . "Tables"}();
    }

    protected function cleanmysqlTables()
    {
        $this->dropCascadeTables();
    }

    protected function cleanpgsqlTables()
    {
        $this->dropCascadeTables();
    }

    protected function cleansqliteTables()
    {
        // do nothing
    }

    protected function dropCascadeTables()
    {
        $manager = DB::getDoctrineSchemaManager();
        $tables = $manager->listTableNames();

        foreach($tables as $table)
        {
            DB::Statement("DROP TABLE " . $table . " CASCADE");
        }
    }

    protected function createTestDbSchema()
    {
        $this->artisan->call('migrate', ["--database" => "testbench", '--path' => '../src/migrations']);
    }

    /**
     * @param       $class_name
     * @param mixed $extra
     * @return array
     */
    protected function make($class_name, $extra = [])
    {
        $created_objs = new Collection();

        while($this->times--)
        {
            $extra_data = ($extra instanceof Closure) ? $extra() : $extra;
            $stub_data = array_merge($this->getModelStub(), $extra_data);
            $created_objs->push($class_name::create($stub_data));
        }

        $this->resetTimes();

        return $created_objs;
    }

    protected function getModelStub()
    {
        throw new BadMethodCallException("You need to implement getModelStub method in your own test class.");
    }

    protected function times($count)
    {
        $this->times = $count;

        return $this;
    }

    protected function resetTimes()
    {
        $this->times = 1;
    }

    protected function assertObjectHasAllAttributes(array $attributes, $object, array $except = [])
    {
        $this->objectHasAllArrayAttributes($attributes, $object, $except);
    }

    protected function objectHasAllArrayAttributes(array $attributes, $object, array $except = [])
    {
        foreach($attributes as $key => $value)
        {
            if(!in_array($key, $except)) $this->assertEquals($value, $object->$key);
        }
    }

    protected function overwriteDatabasePresenceVerifierForTesting()
    {
        App::bindShared('validation.presence', function ($app)
        {
            $verifier = new DatabasePresenceVerifier($this->app['db']);
            $verifier->setConnection('testbench');

            return $verifier;
        });

        $validator = App::make('validator');
        $validator->setPresenceVerifier(App::make('validation.presence'));
    }
}


text:
layout: default
title: "Skills And Training"

---

{% include interests/skills-and-training.md %}

### Who is interested in Skills And Training?

* [Michael Connarty]({{ site.baseurl }}/members/michael-connarty.html)


text:
<?xml version="1.0" encoding="UTF-8" ?>
<!DOCTYPE ldml SYSTEM "../../common/dtd/ldml.dtd">
<!--
Copyright © 1991-2013 Unicode, Inc.
CLDR data files are interpreted according to the LDML specification (http://unicode.org/reports/tr35/)
For terms of use, see http://www.unicode.org/copyright.html
-->
<ldml>
	<identity>
		<version number="$Revision: 8268 $"/>
		<generation date="$Date: 2013-03-01 06:26:02 -0800 (Fri, 01 Mar 2013) $"/>
		<language type="it"/>
	</identity>
	<metadata>
		<casingData>
			<casingItem type="calendar-field">lowercase</casingItem>
			<casingItem type="day-format-except-narrow">lowercase</casingItem>
			<casingItem type="day-narrow">titlecase</casingItem>
			<casingItem type="displayName">titlecase</casingItem>
			<casingItem type="language">lowercase</casingItem>
			<casingItem type="metazone-long">titlecase</casingItem>
			<casingItem type="metazone-short">titlecase</casingItem>
			<casingItem type="month-narrow">titlecase</casingItem>
			<casingItem type="quarter-abbreviated">titlecase</casingItem>
			<casingItem type="script">lowercase</casingItem>
			<casingItem type="symbol">titlecase</casingItem>
			<casingItem type="territory">titlecase</casingItem>
			<casingItem type="type">titlecase</casingItem>
			<casingItem type="variant">lowercase</casingItem>
			<casingItem type="zone-exemplarCity">titlecase</casingItem>
		</casingData>
	</metadata>
</ldml>


text:
/**
 * Created by long on 15-2-10.
 */

var AnimationWithAnchor = BaseScene.extend({
    onEnter:function(){
        this._super();
        var anim = flax.assetsManager.createDisplay(res.animation, "monkey_run", {parent: this, x: cc.visibleRect.center.x, y: cc.visibleRect.center.y});
        anim.play();

        var ui = flax.assetsManager.createDisplay(res.animation, "ui", {parent: this});
        //默认情况下,动画的fps和fla里的设置是一样的
        //In default, an animation's fps is the same as the settings in the fla
        ui["fpsTxt"].text = "FPS: " + anim.fps;

        flax.inputManager.addListener(anim, function(){
            var fps  = anim.fps + 12;
            if(fps > 60) fps = 24;
            anim.fps = fps;
            ui["fpsTxt"].text = "FPS: " + fps;
        });
    }
})


text:
from setuptools import setup, find_packages

args = dict(
    name='Django-bdd',
    version='1.0',
    # declare your packages
    packages=find_packages(exclude=("test",)),
    include_package_data=True,  # declarations in MANIFEST.in
)

setup(**args)


text:
package info.wncwaterfalls.app;

import android.content.ContentValues;
import android.content.Context;
import android.database.Cursor;
import android.database.sqlite.SQLiteDatabase;

import com.readystatesoftware.sqliteasset.SQLiteAssetHelper;

import java.util.Arrays;
import java.util.List;

public class AttrDatabase extends SQLiteAssetHelper {
    private static final String TAG = "AttrDatabase";
    private static final String DATABASE_NAME = "attr";
    private static final int DATABASE_VERSION = 3;

    public static final List<String> COLUMNS = Arrays.asList(
        "_id", "attr_id", "geo_lat", "geo_lon", "name", "alt_names", "description", "height", "stream",
        "landowner", "elevation", "directions", "trail_directions", "trail_difficulty",
        "trail_difficulty_num", "trail_length", "trail_climb", "trail_climb_num",
        "trail_elevationlow", "trail_elevationhigh", "trail_elevationgain", "trail_tread",
        "trail_configuration", "photo", "photo_filename", "map_name", "shared");

    public AttrDatabase(Context context){
        super(context, DATABASE_NAME, null, DATABASE_VERSION);
        this.setForcedUpgrade();
    }

    public Cursor rawQuery(String query, String[] args){
        // Proxy to db.rawQuery
        SQLiteDatabase db = getReadableDatabase();
        Cursor c = db.rawQuery(query, null);
        c.moveToFirst();
        return c;
    }

    public int update(String table, ContentValues values, String whereClause, String[] whereArgs){
        return getReadableDatabase().update(table, values, whereClause, whereArgs);
    }
}


text:
package com.atypon.wayf.data.identity;

import com.fasterxml.jackson.annotation.JsonIgnore;
import com.fasterxml.jackson.annotation.JsonPropertyOrder;

@JsonPropertyOrder({
        "entityId",
        "federationId",
        "type",
        "name",
        "externalIds"
})
public class SamlEntity extends IdentityProvider {

    private String entityId;
    private String federationId;

    public SamlEntity() {
        super();
    }

    public String getEntityId() {
        return entityId;
    }

    public void setEntityId(String entityId) {
        this.entityId = entityId;
    }

    @Override
    @JsonIgnore
    public IdentityProviderType getType() {
        return IdentityProviderType.SAML;
    }

    public String getFederationId() {
        return federationId;
    }

    public void setFederationId(String federationId) {
        this.federationId = federationId;
    }
}


text:
import './models/Rooms';
import './functions/saveStreamingOptions';
import './routes.js';
import './methods.js';
import './settings';


text:
/***************************************************************************//**
 * @file em_bitband.h
 * @brief Bitband Peripheral API
 * @version 4.4.0
 *******************************************************************************
 * @section License
 * <b>Copyright 2016 Silicon Laboratories, Inc. http://www.silabs.com</b>
 *******************************************************************************
 *
 * Permission is granted to anyone to use this software for any purpose,
 * including commercial applications, and to alter it and redistribute it
 * freely, subject to the following restrictions:
 *
 * 1. The origin of this software must not be misrepresented; you must not
 *    claim that you wrote the original software.
 * 2. Altered source versions must be plainly marked as such, and must not be
 *    misrepresented as being the original software.
 * 3. This notice may not be removed or altered from any source distribution.
 *
 * DISCLAIMER OF WARRANTY/LIMITATION OF REMEDIES: Silicon Labs has no
 * obligation to support this Software. Silicon Labs is providing the
 * Software "AS IS", with no express or implied warranties of any kind,
 * including, but not limited to, any implied warranties of merchantability
 * or fitness for any particular purpose or warranties against infringement
 * of any proprietary rights of a third party.
 *
 * Silicon Labs will not be liable for any consequential, incidental, or
 * special damages, or any other relief, or for any claim by any third party,
 * arising from your use of this Software.
 *
 ******************************************************************************/

#ifndef EM_BITBAND_H
#define EM_BITBAND_H

#include "em_bus.h"

#ifdef __cplusplus
extern "C" {
#endif

/***************************************************************************//**
 * @addtogroup emlib
 * @{
 ******************************************************************************/

/***************************************************************************//**
 * @addtogroup BITBAND
 * @brief BITBAND Peripheral API (deprecated - use em_bus.h)
 * @details
 *  API to perform bitbanded accesses to SRAM and peripheral memory.
 * @{
 ******************************************************************************/

/***************************************************************************//**
 * @brief
 *   Perform bit-band operation on peripheral memory location.
 *
 * @details
 *   Bit-banding provides atomic read-modify-write cycle for single bit
 *   modification. Please refer to the reference manual for further details
 *   about bit-banding.
 *
 * @deprecated
 *   Deprecated function. New code should use @ref BUS_RegBitWrite().
 *
 * @note
 *   This function is only atomic on cores which fully support bitbanding.
 *
 * @param[in] addr Peripheral address location to modify bit in.
 *
 * @param[in] bit Bit position to modify, 0-31.
 *
 * @param[in] val Value to set bit to, 0 or 1.
 ******************************************************************************/
#define BITBAND_Peripheral(addr, bit, val) BUS_RegBitWrite(addr, bit, val)

/***************************************************************************//**
 * @brief
 *   Perform a read operation on the peripheral bit-band memory location.
 *
 * @details
 *   This function reads a single bit from the peripheral bit-band alias region.
 *   Bit-banding provides atomic read-modify-write cycle for single bit
 *   modification. Please refer to the reference manual for further details
 *   about bit-banding.
 *
 * @deprecated
 *   Deprecated function. New code should use @ref BUS_RegBitRead().
 *
 * @param[in] addr   Peripheral address location to read.
 *
 * @param[in] bit    Bit position to read, 0-31.
 *
 * @return           Value of the requested bit.
 ******************************************************************************/
#define BITBAND_PeripheralRead(addr, bit) BUS_RegBitRead(addr, bit)

/***************************************************************************//**
 * @brief
 *   Perform bit-band operation on SRAM memory location.
 *
 * @details
 *   Bit-banding provides atomic read-modify-write cycle for single bit
 *   modification. Please refer to the reference manual for further details
 *   about bit-banding.
 *
 * @deprecated
 *   Deprecated function. New code should use @ref BUS_RamBitWrite().
 *
 * @note
 *   This function is only atomic on cores which fully support bitbanding.
 *
 * @param[in] addr SRAM address location to modify bit in.
 *
 * @param[in] bit Bit position to modify, 0-31.
 *
 * @param[in] val Value to set bit to, 0 or 1.
 ******************************************************************************/
#define BITBAND_SRAM(addr, bit, val) BUS_RamBitWrite(addr, bit, val)

/***************************************************************************//**
 * @brief
 *   Read a single bit from the SRAM bit-band alias region.
 *
 * @details
 *   This function reads a single bit from the SRAM bit-band alias region.
 *   Bit-banding provides atomic read-modify-write cycle for single bit
 *   modification. Please refer to the reference manual for further details
 *   about bit-banding.
 *
 * @deprecated
 *   Deprecated function. New code should use @ref BUS_RamBitRead().
 *
 * @param[in] addr    SRAM address location to modify bit in.
 *
 * @param[in] bit     Bit position to modify, 0-31.
 *
 * @return            Value of the requested bit.
 ******************************************************************************/
#define BITBAND_SRAMRead(addr, bit) BUS_RamBitRead(addr, bit)

/** @} (end addtogroup BITBAND) */
/** @} (end addtogroup emlib) */

#ifdef __cplusplus
}
#endif

#endif /* EM_BITBAND_H */


text:
<?= $sTitulo ?>

        <?php
        switch ($sCallMode) {
            case false:
                echo '<form class="form-horizontal" enctype="multipart/form-data" role="form" id="form" method="POST" >';
                break;
            case 'Crea':
                echo '<form class="form-horizontal" enctype="multipart/form-data" role="form" id="form" method="POST" action="' . base_url() . $sControlador . '/Crea">';
                break;
            case 'Modifica':
                echo '<form class="form-horizontal" enctype="multipart/form-data" role="form" id="form" method="POST" action="' . base_url() . $sControlador . '/Modifica">';
                break;
        }
        ?>

                Nro. Documento

                <input type="text" class="form-control k-textbox" id="DOC_CLIENTE" name="DOC_EMPLEADO" placeholder="Identificación" >

                Ciudad

                <select class="form-control k-dropdown" id='CIUDAD_ID' name='CIUDAD_ID'>
                </select>

                Nombres

                <input type="text" class="form-control k-textbox" id="NOMBRES" placeholder="Nombres Completos" name="NOMBRES">

                Apellidos

                <input type="text" class="form-control k-textbox" id="APELLIDOS" name="APELLIDOS" placeholder="Apellidos Completos" >

                Dirección

                <input type="text" class="form-control k-textbox" id="DIRECCION" placeholder="Dirección" name="DIRECCION">

                Fecha nacimiento

                <input type="text" id='FECHA_NACIMIENTO' class="form-control k-textbox "  name="FECHA_NACIMIENTO" placeholder="Fecha nacimiento" >

                Télefono

                <input type="text" class="form-control k-textbox" id="TELEFONO" placeholder="Tel" name="TELEFONO">

                Correo Elecctronico

                <input type="email" class="form-control k-textbox" id="TELEFONO" placeholder="Correo Electronico" name="E_MAIL">

                Estado Civil

                <select class="k-widget k-dropdown k-header form-control" id="ESTADO_CIVIL">
                    <option>Soltero/a</option>
                    <option>Comprometido/a</option>
                    <option>Casado/a</option>
                    <option>Divorciado/a</option>
                    <option>Viudo/a</option>
                </select>

                Nro. de Hijos

                <input type="number" class="form-control k-textbox" id="CANT_HIJOS" name="CANT_HIJOS" >

                Nivel Academico
                 <select class="k-widget k-dropdown k-header form-control" name="NIVEL_ACADEMICO" id="NIVEL_ACADEMICO">
                </select>

                <input type="text" class="form-control k-textbox" id="NIVEL_ACADEMICO"  name="NIVEL_ACADEMICO" placeholder="Horario Laboral">

                Tipo Empleado

                <select class="k-widget k-dropdown k-header form-control" name="TIPO_EMPLEADO" id="TIPO_EMPLEADO">
                </select>

                Foto

                <input name="RUTA_FOTO" id="Foto" type="file" />

                Huella

                <input name="RUTA_HUELLA" id="Huella" type="file" />

                Observaciones

                <textarea class="form-control k-textbox" id="OBSERVACIONES" name="OBSERVACIONES" ></textarea>

            Cancelar
            <button class="k-button k-state-default"><?= ($sCallMode == 'Crea') ? 'Guardar' : 'Modificar'; ?></button>

        </form>


text:
from heapq import *
class MedianFinder:
    def __init__(self):
        # self.maxHeap stores the lower half and self.minHeap stores the upper half
        self.maxHeap, self.minHeap = [], []

    def addNum(self, num: int) -> None:
        # try to add the number in self.maxHeap first
        if not self.maxHeap or -num >= self.maxHeap[0]:
            heappush(self.maxHeap, -num)
        else:
            heappush(self.minHeap, num)

        # if there are more elements than expected then balance
        if len(self.maxHeap) > len(self.minHeap) + 1:
            out = -heappop(self.maxHeap)
            heappush(self.minHeap, out)
        elif len(self.maxHeap) < len(self.minHeap):
            out = heappop(self.minHeap)
            heappush(self.maxHeap, -out)

    def findMedian(self) -> float:
        if len(self.maxHeap) == len(self.minHeap):
            return (-self.maxHeap[0] + self.minHeap[0]) / 2

        return -self.maxHeap[0]

ob = MedianFinder()
ob.addNum(1)
print(ob.findMedian())
ob.addNum(2)
print(ob.findMedian())
ob.addNum(3)
print(ob.findMedian())