text:
VERSION_NUMBER=$1
SQL_CONNECTION=$2
INCLUDE_RELEASE_FOLDER=$3

{
if [ -z "${VERSION_NUMBER}" ]; then
  echo "VERSION_NUMBER (parameter 1) is not defined"
  exit 0
fi
if [ -z "${SQL_CONNECTION}" ]; then
	echo "SQL_CONNECTION (parameter 2) is not defined"
	exit 0
fi

#91: Allow option to include release folder or not. Useful for developers, but should be excluded by default
#Upper value
INCLUDE_RELEASE_FOLDER=$(echo $INCLUDE_RELEASE_FOLDER | awk '{print toupper($0)}')
if [ "$INCLUDE_RELEASE_FOLDER" = "" ]; then
	INCLUDE_RELEASE_FOLDER=N
fi
echo "INCLUDE_RELEASE_FOLDER: $INCLUDE_RELEASE_FOLDER"
}

echo "Building release $VERSION_NUMBER"

#Generate no_op code
START_DIR="$PWD"
cd ../source/packages
sqlplus $SQL_CONNECTION @../gen_no_op.sql
cd $START_DIR

#*** VARIABLES ***
RELEASE_FOLDER=../releases/$VERSION_NUMBER
INSTALL=$RELEASE_FOLDER/"logger_install.sql"
NO_OP=$RELEASE_FOLDER/"logger_no_op.sql"

#Clear release folder (if it exists) and make directory
rm -rf ../releases/$VERSION_NUMBER
mkdir ../releases/$VERSION_NUMBER

#Build files

#rm -f ../build/logger_install.sql
#rm -f ../build/logger_latest.zip
#rm -f ../build/logger_no_op.sql

#PREINSTALL
cat ../source/install/logger_install_prereqs.sql > $INSTALL
printf '\n' >> $INSTALL

#NO OP Code
printf "\x2d\x2d This file installs a NO-OP version of the logger package that has all of the same procedures and functions,\n" > $NO_OP
printf "\x2d\x2d but does not actually write to any tables. Additionally, it has no other object dependencies.\n" >> $NO_OP
printf "\x2d\x2d You can review the documentation at https://github.com/OraOpenSource/Logger for more information.\n" >> $NO_OP
printf '\n' >> $NO_OP
#Seeting compilation flag so that tables are just skeletons (no triggers, sequences, etc)
printf "alter session set plsql_ccflags='logger_no_op_install:true' \n/ \n\n\n" >> $NO_OP

#78: Need to build tables for no_op to reference
#TABLES - Output for both regular and NO_OP
printf 'PROMPT tables/logger_logs.sql \n' | tee -a $INSTALL $NO_OP > /dev/null
cat ../source/tables/logger_logs.sql | tee -a $INSTALL $NO_OP > /dev/null
printf '\n' | tee -a $INSTALL $NO_OP > /dev/null
printf 'PROMPT tables/logger_prefs.sql \n' | tee -a $INSTALL $NO_OP > /dev/null
cat ../source/tables/logger_prefs.sql | tee -a $INSTALL $NO_OP > /dev/null
printf '\n' | tee -a $INSTALL $NO_OP > /dev/null
printf 'PROMPT tables/logger_logs_apex_items.sql \n' | tee -a $INSTALL $NO_OP > /dev/null
cat ../source/tables/logger_logs_apex_items.sql | tee -a $INSTALL $NO_OP > /dev/null
printf '\n' | tee -a $INSTALL $NO_OP > /dev/null
printf 'PROMPT tables/logger_prefs_by_client_id.sql \n' | tee -a $INSTALL $NO_OP > /dev/null
cat ../source/tables/logger_prefs_by_client_id.sql | tee -a $INSTALL $NO_OP > /dev/null
printf '\n' | tee -a $INSTALL $NO_OP > /dev/null

#JOBS
printf 'PROMPT jobs/logger_purge_job.sql \n' >> $INSTALL
cat ../source/jobs/logger_purge_job.sql >> $INSTALL
printf '\n' >> $INSTALL
printf 'PROMPT jobs/logger_unset_prefs_by_client.sql \n' >> $INSTALL
cat ../source/jobs/logger_unset_prefs_by_client.sql >> $INSTALL
printf '\n' >> $INSTALL

#VIEWS - Output for both regular and NO_OP
printf 'PROMPT views/logger_logs_5_min.sql \n' | tee -a $INSTALL $NO_OP > /dev/null
cat ../source/views/logger_logs_5_min.sql | tee -a $INSTALL $NO_OP > /dev/null
printf '\n' | tee -a $INSTALL $NO_OP > /dev/null
printf 'PROMPT views/logger_logs_60_min.sql \n' | tee -a $INSTALL $NO_OP > /dev/null
cat ../source/views/logger_logs_60_min.sql | tee -a $INSTALL $NO_OP > /dev/null
printf '\n' | tee -a $INSTALL $NO_OP > /dev/null
printf 'PROMPT views/logger_logs_terse.sql\n' | tee -a $INSTALL $NO_OP > /dev/null
cat ../source/views/logger_logs_terse.sql | tee -a $INSTALL $NO_OP > /dev/null
printf '\n' | tee -a $INSTALL $NO_OP > /dev/null

#PACKAGES
printf 'PROMPT packages/logger.pks \n' >> $INSTALL
cat ../source/packages/logger.pks >> $INSTALL
printf '\n' >> $INSTALL
printf 'PROMPT packages/logger.pkb \n' >> $INSTALL
cat ../source/packages/logger.pkb >> $INSTALL
printf '\n' >> $INSTALL

#Recompile logger_prefs trigger as it has dependencies on logger.pks
printf 'PROMPT Recompile biu_logger_prefs after logger.pkb \n' >> $INSTALL
printf '\nalter trigger biu_logger_prefs compile;\n' | tee -a $INSTALL $NO_OP > /dev/null

#CONTEXTS
printf 'PROMPT contexts/logger_context.sql \n' >> $INSTALL
cat ../source/contexts/logger_context.sql >> $INSTALL
printf '\n' >> $INSTALL

#PROCEDURES
printf 'PROMPT procedures/logger_configure.plb \n' >> $INSTALL
cat ../source/procedures/logger_configure.plb >> $INSTALL
printf '\n' >> $INSTALL

#Post install
printf 'PROMPT install/post_install_configuration.sql \n' >> $INSTALL
cat ../source/install/post_install_configuration.sql >> $INSTALL
printf '\n' >> $INSTALL

#NO OP Code
printf '\n\nprompt *** logger.pks *** \n\n' >> $NO_OP
cat ../source/packages/logger.pks >> $NO_OP
printf '\n\nprompt *** logger.pkb *** \n\n' >> $NO_OP
cat ../source/packages/logger_no_op.pkb >> $NO_OP
printf '\n\nprompt\n' >> $NO_OP
printf 'prompt *************************************************\n' >> $NO_OP
printf 'prompt Now executing LOGGER.STATUS...\n' >> $NO_OP
printf 'prompt ' >> $NO_OP
printf '\nbegin \n\tlogger.status; \nend;\n/\n\n' >> $NO_OP
printf 'prompt *************************************************\n' >> $NO_OP
printf '\n\n' >> $NO_OP

#Recompile logger_logs_terse since it depends on logger
printf '\nalter view logger_logs_terse compile;\n' | tee -a $INSTALL $NO_OP > /dev/null

#Copy "other" scripts
cp -f ../source/install/create_user.sql $RELEASE_FOLDER
cp -f ../source/install/drop_logger.sql $RELEASE_FOLDER

#Copy Scripts
cp -r ../source/scripts $RELEASE_FOLDER

#Copy main package file for developers to easily review
cp -f ../source/packages/logger.* $RELEASE_FOLDER

#Copy README
cp -f ../README.md $RELEASE_FOLDER

#Copy demo scripts
cp -fr ../demos $RELEASE_FOLDER

#Copy docs #89
cp -fr ../docs $RELEASE_FOLDER

#Copy License
cp -f ../LICENSE $RELEASE_FOLDER

chmod 777 $RELEASE_FOLDER/*.*

#Replace any references for the version number
sed -i.del "s/x\.x\.x/$VERSION_NUMBER/g" $RELEASE_FOLDER/logger_install.sql

# Orginial:
# sed -i.del "s/x\.x\.x/$VERSION_NUMBER/g" $RELEASE_FOLDER/logger.pks
# #129: Change the version number to the individual components
echo $VERSION_NUMBER | while IFS=. read major minor patch beta
do
   #Replace major version
   sed -i.del "s/-91/$major/g" $RELEASE_FOLDER/logger.pks
   sed -i.del "s/-91/$major/g" $RELEASE_FOLDER/logger_install.sql
   #Replace minor version
   sed -i.del "s/-92/$minor/g" $RELEASE_FOLDER/logger.pks
   sed -i.del "s/-92/$minor/g" $RELEASE_FOLDER/logger_install.sql
   #Replace patch version
   sed -i.del "s/-93/$patch/g" $RELEASE_FOLDER/logger.pks
   sed -i.del "s/-93/$patch/g" $RELEASE_FOLDER/logger_install.sql
   #Replace #beta# version
   sed -i.del "s/#beta#/$beta/g" $RELEASE_FOLDER/logger.pks
   sed -i.del "s/#beta#/$beta/g" $RELEASE_FOLDER/logger_install.sql
done

#need to remove the backup file required for sed call
rm -rf $RELEASE_FOLDER/*.del

#Old windows zip7za a -tzip $/logger_$VERSION_NUMBER.zip ../build/*.sql ../build/*.html
#By CDing into the release_folder we don't get the full path in the zip file
cd $RELEASE_FOLDER
zip -r logger_$VERSION_NUMBER.zip .

#91: Copy zip to release root
cp -f logger_$VERSION_NUMBER.zip ../.

#Remove release folder if appliable
if [ "$INCLUDE_RELEASE_FOLDER" != "Y" ]; then
  echo Removing release folder
  cd $START_DIR
  rm -rf $RELEASE_FOLDER
else
  echo Keeping release folder
fi


text: Bandango::Application.config.secret_key_base = ENV["SECRET_BASE"]

redpajama_set_name: RedPajamaGithub


text:
<?php
namespace Craft;

class DropDownDate_DateInputModel extends BaseModel
{
    /**
     * @access protected
     * @return array
     */
    protected function defineAttributes()
    {
        return array(
            'yearRangeStart' => array(AttributeType::String, 'default' => '-115 year', 'required' => true),
            'yearRangeEnd' => array(AttributeType::String, 'default' => 'now', 'required' => true)
        );
    }

    /**
     * @param null $attributes
     * @param bool $clearErrors
     * @return bool|void
     */
    public function validate($attributes = null, $clearErrors = true)
    {
        parent::validate($attributes, $clearErrors);

        if ($this->yearRangeStart)
        {
            if (!is_numeric($this->yearRangeStart) && strtotime($this->yearRangeStart) === false)
            {
                $this->addError('yearRangeStart', Craft::t('Not a valid value, use a full year notation or a strtotime() value.'));
            }
        }

        if ($this->yearRangeEnd)
        {
            if (!is_numeric($this->yearRangeEnd) && strtotime($this->yearRangeEnd) === false)
            {
                $this->addError('yearRangeEnd', Craft::t('Not a valid value, use a full year notation or a strtotime() value.'));
            }
        }

        return !$this->hasErrors();
    }
}


text:
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading;

namespace Cosmos.Assembler {
  public class ElementReference {
    private static Dictionary<string, ElementReference> mLookup = new Dictionary<string, ElementReference>(StringComparer.InvariantCultureIgnoreCase);
    public static ElementReference New(string aName, int aOffset) {
      ElementReference xResult;
      if (aName.StartsWith(".")) {
        aName = Label.LastFullLabel + aName;
      }
      string xId = String.Intern(aName + "@@" + aOffset);
      if (mLookup.TryGetValue(xId, out xResult)) {
        return xResult;
      }
      if (aOffset == 0) {
        xResult = new ElementReference(aName);
      } else {
        xResult = new ElementReference(aName, aOffset);
      }
      mLookup.Add(xId, xResult);
      return xResult;
    }

    public static ElementReference New(string aName) {
      return New(aName, 0);
    }

    private ElementReference(string aName, int aOffset)
      : this(aName) {
      Offset = aOffset;
    }

    private ElementReference(string aName) {
      if (String.IsNullOrEmpty(aName)) {
        throw new ArgumentNullException("aName");
      }
      if (aName == "00h") {
        Console.Write("");
      }
      if (aName.StartsWith(".")) {
        Name = Label.LastFullLabel + aName;
      } else {
        Name = aName;
      }
    }

    private ulong? mActualAddress;

    private static ReaderWriterLockSlim mCacheLocker = new ReaderWriterLockSlim(LockRecursionPolicy.NoRecursion);
    private static Dictionary<string, BaseAssemblerElement> mCache;// = new SortedList<string, BaseAssemblerElement>(StringComparer.InvariantCultureIgnoreCase);
    private static int? mThreadId = null;

    private static BaseAssemblerElement DoResolve(Cosmos.Assembler.Assembler aAssembler, string aName) {
      if (!mThreadId.HasValue) {
        mThreadId = Thread.CurrentThread.ManagedThreadId;
      } else {
        if (mThreadId.Value != Thread.CurrentThread.ManagedThreadId) {
          throw new Exception("Called from multiple threads");
        }
      }
      mCacheLocker.EnterReadLock();
      try {
        if (mCache != null) {
          BaseAssemblerElement xTempResult;
          if (mCache.TryGetValue(aName, out xTempResult)) {
            return xTempResult;
          }
        }
      } finally {
        mCacheLocker.ExitReadLock();
      }
      mCacheLocker.EnterWriteLock();
      try {
        if (mCache == null) {
          mCache = new Dictionary<string, BaseAssemblerElement>(StringComparer.InvariantCultureIgnoreCase);
          int xMax = aAssembler.AllAssemblerElementCount;
          for (int i = 0; i < xMax; i++) {
            var xInstruction = aAssembler.GetAssemblerElement(i);
            var xLabel = xInstruction as Label;
            if (xLabel != null) {
              mCache.Add(xLabel.QualifiedName, xLabel);
            }
            var xDataMember = xInstruction as DataMember;
            if (xDataMember != null) {
              if (mCache.ContainsKey(xDataMember.Name)) {
                Console.Write("");
              }
              mCache.Add(xDataMember.Name, xDataMember);
            }
          }
        }
        BaseAssemblerElement xTempResult;
        if (mCache.TryGetValue(aName, out xTempResult)) {
          return xTempResult;
        }
        throw new Exception("Cannot resolve ElementReference to '" + aName + "'!");
        //foreach(var xInstruction in aAssembler.Instructions ) {
        //    var xLabel = xInstruction as Label;
        //    if(xLabel!=null) {
        //        if(aName.Equals(xLabel.Name, StringComparison.InvariantCultureIgnoreCase)) {
        //            xTempResult = xLabel;
        //            break;
        //        }
        //    }
        //}
        //if (xTempResult == null) {
        //    foreach (var xDataMember in aAssembler.DataMembers) {
        //        if (aName.Equals(xDataMember.Name, StringComparison.InvariantCultureIgnoreCase)) {
        //            xTempResult = xDataMember;
        //            break;
        //        }
        //    }
        //}
      } finally {
        mCacheLocker.ExitWriteLock();
      }
    }

    public bool Resolve(Cosmos.Assembler.Assembler aAssembler, out ulong aAddress) {
      //
      if (mActualAddress != null) {
        aAddress = mActualAddress.Value;
        return true;
      }
      var xElement = DoResolve(aAssembler, Name);
      if (xElement != null) {
        if (xElement.ActualAddress.HasValue) {
          mActualAddress = (ulong)((long)xElement.ActualAddress.Value + Offset);
          aAddress = mActualAddress.Value;
          return true;
        }
      }

      aAddress = 0;
      return false;
    }

    public int Offset {
      get;
      set;
    }

    public string Name {
      get;
      set;
    }

    public override string ToString() {
      if (Offset != 0) {
        return Name + " + " + Offset;
      } else {
        return Name;
      }
    }
  }
}


text:
class Song < ActiveRecord::Base
  has_attached_file :mp3
  has_attached_file :chord_chart

  validates :title,
            presence: true
  validates_attachment_content_type :mp3, :content_type => ['audio/mp3', 'audio/ogg']
  validates_attachment_content_type :chord_chart, :content_type => ['application/pdf']

end


text:
@extends('layouts.app')

@section('title', 'Edit newsletter')

@section('content')

Newsletters

Edit newsletter

            @include('flash::message')
            {!! Form::model($newsletter, ['route' => ['newsletters.update', $newsletter], 'method' => 'PATCH']) !!}
            @include('newsletters._form')
            {!! Form::close() !!}

@endsection


text:
using Microsoft.EntityFrameworkCore.Metadata;
using Microsoft.EntityFrameworkCore.Migrations;
using System;
using System.Collections.Generic;

namespace IdentityManagerDotNet.Migrations
{
    public partial class InitialConfigurationDbContext : Migration
    {
        protected override void Up(MigrationBuilder migrationBuilder)
        {
            migrationBuilder.CreateTable(
                name: "ApiResources",
                columns: table => new
                {
                    Id = table.Column<int>(type: "int", nullable: false)
                        .Annotation("SqlServer:ValueGenerationStrategy", SqlServerValueGenerationStrategy.IdentityColumn),
                    Description = table.Column<string>(type: "nvarchar(1000)", maxLength: 1000, nullable: true),
                    DisplayName = table.Column<string>(type: "nvarchar(200)", maxLength: 200, nullable: true),
                    Enabled = table.Column<bool>(type: "bit", nullable: false),
                    Name = table.Column<string>(type: "nvarchar(200)", maxLength: 200, nullable: false)
                },
                constraints: table =>
                {
                    table.PrimaryKey("PK_ApiResources", x => x.Id);
                });

            migrationBuilder.CreateTable(
                name: "Clients",
                columns: table => new
                {
                    Id = table.Column<int>(type: "int", nullable: false)
                        .Annotation("SqlServer:ValueGenerationStrategy", SqlServerValueGenerationStrategy.IdentityColumn),
                    AbsoluteRefreshTokenLifetime = table.Column<int>(type: "int", nullable: false),
                    AccessTokenLifetime = table.Column<int>(type: "int", nullable: false),
                    AccessTokenType = table.Column<int>(type: "int", nullable: false),
                    AllowAccessTokensViaBrowser = table.Column<bool>(type: "bit", nullable: false),
                    AllowOfflineAccess = table.Column<bool>(type: "bit", nullable: false),
                    AllowPlainTextPkce = table.Column<bool>(type: "bit", nullable: false),
                    AllowRememberConsent = table.Column<bool>(type: "bit", nullable: false),
                    AlwaysIncludeUserClaimsInIdToken = table.Column<bool>(type: "bit", nullable: false),
                    AlwaysSendClientClaims = table.Column<bool>(type: "bit", nullable: false),
                    AuthorizationCodeLifetime = table.Column<int>(type: "int", nullable: false),
                    BackChannelLogoutSessionRequired = table.Column<bool>(type: "bit", nullable: false),
                    BackChannelLogoutUri = table.Column<string>(type: "nvarchar(max)", nullable: true),
                    ClientId = table.Column<string>(type: "nvarchar(200)", maxLength: 200, nullable: false),
                    ClientName = table.Column<string>(type: "nvarchar(200)", maxLength: 200, nullable: true),
                    ClientUri = table.Column<string>(type: "nvarchar(2000)", maxLength: 2000, nullable: true),
                    ConsentLifetime = table.Column<int>(type: "int", nullable: true),
                    Description = table.Column<string>(type: "nvarchar(1000)", maxLength: 1000, nullable: true),
                    EnableLocalLogin = table.Column<bool>(type: "bit", nullable: false),
                    Enabled = table.Column<bool>(type: "bit", nullable: false),
                    FrontChannelLogoutSessionRequired = table.Column<bool>(type: "bit", nullable: false),
                    FrontChannelLogoutUri = table.Column<string>(type: "nvarchar(max)", nullable: true),
                    IdentityTokenLifetime = table.Column<int>(type: "int", nullable: false),
                    IncludeJwtId = table.Column<bool>(type: "bit", nullable: false),
                    LogoUri = table.Column<string>(type: "nvarchar(max)", nullable: true),
                    NormalizedClientId = table.Column<string>(type: "nvarchar(max)", nullable: true),
                    PrefixClientClaims = table.Column<bool>(type: "bit", nullable: false),
                    ProtocolType = table.Column<string>(type: "nvarchar(200)", maxLength: 200, nullable: false),
                    RefreshTokenExpiration = table.Column<int>(type: "int", nullable: false),
                    RefreshTokenUsage = table.Column<int>(type: "int", nullable: false),
                    RequireClientSecret = table.Column<bool>(type: "bit", nullable: false),
                    RequireConsent = table.Column<bool>(type: "bit", nullable: false),
                    RequirePkce = table.Column<bool>(type: "bit", nullable: false),
                    SlidingRefreshTokenLifetime = table.Column<int>(type: "int", nullable: false),
                    UpdateAccessTokenClaimsOnRefresh = table.Column<bool>(type: "bit", nullable: false)
                },
                constraints: table =>
                {
                    table.PrimaryKey("PK_Clients", x => x.Id);
                });

            migrationBuilder.CreateTable(
                name: "IdentityResources",
                columns: table => new
                {
                    Id = table.Column<int>(type: "int", nullable: false)
                        .Annotation("SqlServer:ValueGenerationStrategy", SqlServerValueGenerationStrategy.IdentityColumn),
                    Description = table.Column<string>(type: "nvarchar(1000)", maxLength: 1000, nullable: true),
                    DisplayName = table.Column<string>(type: "nvarchar(200)", maxLength: 200, nullable: true),
                    Emphasize = table.Column<bool>(type: "bit", nullable: false),
                    Enabled = table.Column<bool>(type: "bit", nullable: false),
                    Name = table.Column<string>(type: "nvarchar(200)", maxLength: 200, nullable: false),
                    Required = table.Column<bool>(type: "bit", nullable: false),
                    ShowInDiscoveryDocument = table.Column<bool>(type: "bit", nullable: false)
                },
                constraints: table =>
                {
                    table.PrimaryKey("PK_IdentityResources", x => x.Id);
                });

            migrationBuilder.CreateTable(
                name: "ApiClaims",
                columns: table => new
                {
                    Id = table.Column<int>(type: "int", nullable: false)
                        .Annotation("SqlServer:ValueGenerationStrategy", SqlServerValueGenerationStrategy.IdentityColumn),
                    ApiResourceId = table.Column<int>(type: "int", nullable: false),
                    Type = table.Column<string>(type: "nvarchar(200)", maxLength: 200, nullable: false)
                },
                constraints: table =>
                {
                    table.PrimaryKey("PK_ApiClaims", x => x.Id);
                    table.ForeignKey(
                        name: "FK_ApiClaims_ApiResources_ApiResourceId",
                        column: x => x.ApiResourceId,
                        principalTable: "ApiResources",
                        principalColumn: "Id",
                        onDelete: ReferentialAction.Cascade);
                });

            migrationBuilder.CreateTable(
                name: "ApiScopes",
                columns: table => new
                {
                    Id = table.Column<int>(type: "int", nullable: false)
                        .Annotation("SqlServer:ValueGenerationStrategy", SqlServerValueGenerationStrategy.IdentityColumn),
                    ApiResourceId = table.Column<int>(type: "int", nullable: false),
                    Description = table.Column<string>(type: "nvarchar(1000)", maxLength: 1000, nullable: true),
                    DisplayName = table.Column<string>(type: "nvarchar(200)", maxLength: 200, nullable: true),
                    Emphasize = table.Column<bool>(type: "bit", nullable: false),
                    Name = table.Column<string>(type: "nvarchar(200)", maxLength: 200, nullable: false),
                    Required = table.Column<bool>(type: "bit", nullable: false),
                    ShowInDiscoveryDocument = table.Column<bool>(type: "bit", nullable: false)
                },
                constraints: table =>
                {
                    table.PrimaryKey("PK_ApiScopes", x => x.Id);
                    table.ForeignKey(
                        name: "FK_ApiScopes_ApiResources_ApiResourceId",
                        column: x => x.ApiResourceId,
                        principalTable: "ApiResources",
                        principalColumn: "Id",
                        onDelete: ReferentialAction.Cascade);
                });

            migrationBuilder.CreateTable(
                name: "ApiSecrets",
                columns: table => new
                {
                    Id = table.Column<int>(type: "int", nullable: false)
                        .Annotation("SqlServer:ValueGenerationStrategy", SqlServerValueGenerationStrategy.IdentityColumn),
                    ApiResourceId = table.Column<int>(type: "int", nullable: false),
                    Description = table.Column<string>(type: "nvarchar(1000)", maxLength: 1000, nullable: true),
                    Expiration = table.Column<DateTime>(type: "datetime2", nullable: true),
                    Type = table.Column<string>(type: "nvarchar(250)", maxLength: 250, nullable: true),
                    Value = table.Column<string>(type: "nvarchar(2000)", maxLength: 2000, nullable: true)
                },
                constraints: table =>
                {
                    table.PrimaryKey("PK_ApiSecrets", x => x.Id);
                    table.ForeignKey(
                        name: "FK_ApiSecrets_ApiResources_ApiResourceId",
                        column: x => x.ApiResourceId,
                        principalTable: "ApiResources",
                        principalColumn: "Id",
                        onDelete: ReferentialAction.Cascade);
                });

            migrationBuilder.CreateTable(
                name: "ClientClaims",
                columns: table => new
                {
                    Id = table.Column<int>(type: "int", nullable: false)
                        .Annotation("SqlServer:ValueGenerationStrategy", SqlServerValueGenerationStrategy.IdentityColumn),
                    ClientId = table.Column<int>(type: "int", nullable: false),
                    Type = table.Column<string>(type: "nvarchar(250)", maxLength: 250, nullable: false),
                    Value = table.Column<string>(type: "nvarchar(250)", maxLength: 250, nullable: false)
                },
                constraints: table =>
                {
                    table.PrimaryKey("PK_ClientClaims", x => x.Id);
                    table.ForeignKey(
                        name: "FK_ClientClaims_Clients_ClientId",
                        column: x => x.ClientId,
                        principalTable: "Clients",
                        principalColumn: "Id",
                        onDelete: ReferentialAction.Cascade);
                });

            migrationBuilder.CreateTable(
                name: "ClientCorsOrigins",
                columns: table => new
                {
                    Id = table.Column<int>(type: "int", nullable: false)
                        .Annotation("SqlServer:ValueGenerationStrategy", SqlServerValueGenerationStrategy.IdentityColumn),
                    ClientId = table.Column<int>(type: "int", nullable: false),
                    Origin = table.Column<string>(type: "nvarchar(150)", maxLength: 150, nullable: false)
                },
                constraints: table =>
                {
                    table.PrimaryKey("PK_ClientCorsOrigins", x => x.Id);
                    table.ForeignKey(
                        name: "FK_ClientCorsOrigins_Clients_ClientId",
                        column: x => x.ClientId,
                        principalTable: "Clients",
                        principalColumn: "Id",
                        onDelete: ReferentialAction.Cascade);
                });

            migrationBuilder.CreateTable(
                name: "ClientGrantTypes",
                columns: table => new
                {
                    Id = table.Column<int>(type: "int", nullable: false)
                        .Annotation("SqlServer:ValueGenerationStrategy", SqlServerValueGenerationStrategy.IdentityColumn),
                    ClientId = table.Column<int>(type: "int", nullable: false),
                    GrantType = table.Column<string>(type: "nvarchar(250)", maxLength: 250, nullable: false)
                },
                constraints: table =>
                {
                    table.PrimaryKey("PK_ClientGrantTypes", x => x.Id);
                    table.ForeignKey(
                        name: "FK_ClientGrantTypes_Clients_ClientId",
                        column: x => x.ClientId,
                        principalTable: "Clients",
                        principalColumn: "Id",
                        onDelete: ReferentialAction.Cascade);
                });

            migrationBuilder.CreateTable(
                name: "ClientIdPRestrictions",
                columns: table => new
                {
                    Id = table.Column<int>(type: "int", nullable: false)
                        .Annotation("SqlServer:ValueGenerationStrategy", SqlServerValueGenerationStrategy.IdentityColumn),
                    ClientId = table.Column<int>(type: "int", nullable: false),
                    Provider = table.Column<string>(type: "nvarchar(200)", maxLength: 200, nullable: false)
                },
                constraints: table =>
                {
                    table.PrimaryKey("PK_ClientIdPRestrictions", x => x.Id);
                    table.ForeignKey(
                        name: "FK_ClientIdPRestrictions_Clients_ClientId",
                        column: x => x.ClientId,
                        principalTable: "Clients",
                        principalColumn: "Id",
                        onDelete: ReferentialAction.Cascade);
                });

            migrationBuilder.CreateTable(
                name: "ClientPostLogoutRedirectUris",
                columns: table => new
                {
                    Id = table.Column<int>(type: "int", nullable: false)
                        .Annotation("SqlServer:ValueGenerationStrategy", SqlServerValueGenerationStrategy.IdentityColumn),
                    ClientId = table.Column<int>(type: "int", nullable: false),
                    PostLogoutRedirectUri = table.Column<string>(type: "nvarchar(2000)", maxLength: 2000, nullable: false)
                },
                constraints: table =>
                {
                    table.PrimaryKey("PK_ClientPostLogoutRedirectUris", x => x.Id);
                    table.ForeignKey(
                        name: "FK_ClientPostLogoutRedirectUris_Clients_ClientId",
                        column: x => x.ClientId,
                        principalTable: "Clients",
                        principalColumn: "Id",
                        onDelete: ReferentialAction.Cascade);
                });

            migrationBuilder.CreateTable(
                name: "ClientProperties",
                columns: table => new
                {
                    Id = table.Column<int>(type: "int", nullable: false)
                        .Annotation("SqlServer:ValueGenerationStrategy", SqlServerValueGenerationStrategy.IdentityColumn),
                    ClientId = table.Column<int>(type: "int", nullable: false),
                    Key = table.Column<string>(type: "nvarchar(250)", maxLength: 250, nullable: false),
                    Value = table.Column<string>(type: "nvarchar(2000)", maxLength: 2000, nullable: false)
                },
                constraints: table =>
                {
                    table.PrimaryKey("PK_ClientProperties", x => x.Id);
                    table.ForeignKey(
                        name: "FK_ClientProperties_Clients_ClientId",
                        column: x => x.ClientId,
                        principalTable: "Clients",
                        principalColumn: "Id",
                        onDelete: ReferentialAction.Cascade);
                });

            migrationBuilder.CreateTable(
                name: "ClientRedirectUris",
                columns: table => new
                {
                    Id = table.Column<int>(type: "int", nullable: false)
                        .Annotation("SqlServer:ValueGenerationStrategy", SqlServerValueGenerationStrategy.IdentityColumn),
                    ClientId = table.Column<int>(type: "int", nullable: false),
                    RedirectUri = table.Column<string>(type: "nvarchar(2000)", maxLength: 2000, nullable: false)
                },
                constraints: table =>
                {
                    table.PrimaryKey("PK_ClientRedirectUris", x => x.Id);
                    table.ForeignKey(
                        name: "FK_ClientRedirectUris_Clients_ClientId",
                        column: x => x.ClientId,
                        principalTable: "Clients",
                        principalColumn: "Id",
                        onDelete: ReferentialAction.Cascade);
                });

            migrationBuilder.CreateTable(
                name: "ClientScopes",
                columns: table => new
                {
                    Id = table.Column<int>(type: "int", nullable: false)
                        .Annotation("SqlServer:ValueGenerationStrategy", SqlServerValueGenerationStrategy.IdentityColumn),
                    ClientId = table.Column<int>(type: "int", nullable: false),
                    Scope = table.Column<string>(type: "nvarchar(200)", maxLength: 200, nullable: false)
                },
                constraints: table =>
                {
                    table.PrimaryKey("PK_ClientScopes", x => x.Id);
                    table.ForeignKey(
                        name: "FK_ClientScopes_Clients_ClientId",
                        column: x => x.ClientId,
                        principalTable: "Clients",
                        principalColumn: "Id",
                        onDelete: ReferentialAction.Cascade);
                });

            migrationBuilder.CreateTable(
                name: "ClientSecrets",
                columns: table => new
                {
                    Id = table.Column<int>(type: "int", nullable: false)
                        .Annotation("SqlServer:ValueGenerationStrategy", SqlServerValueGenerationStrategy.IdentityColumn),
                    ClientId = table.Column<int>(type: "int", nullable: false),
                    Description = table.Column<string>(type: "nvarchar(2000)", maxLength: 2000, nullable: true),
                    Expiration = table.Column<DateTime>(type: "datetime2", nullable: true),
                    Type = table.Column<string>(type: "nvarchar(250)", maxLength: 250, nullable: true),
                    Value = table.Column<string>(type: "nvarchar(2000)", maxLength: 2000, nullable: false)
                },
                constraints: table =>
                {
                    table.PrimaryKey("PK_ClientSecrets", x => x.Id);
                    table.ForeignKey(
                        name: "FK_ClientSecrets_Clients_ClientId",
                        column: x => x.ClientId,
                        principalTable: "Clients",
                        principalColumn: "Id",
                        onDelete: ReferentialAction.Cascade);
                });

            migrationBuilder.CreateTable(
                name: "IdentityClaims",
                columns: table => new
                {
                    Id = table.Column<int>(type: "int", nullable: false)
                        .Annotation("SqlServer:ValueGenerationStrategy", SqlServerValueGenerationStrategy.IdentityColumn),
                    IdentityResourceId = table.Column<int>(type: "int", nullable: false),
                    Type = table.Column<string>(type: "nvarchar(200)", maxLength: 200, nullable: false)
                },
                constraints: table =>
                {
                    table.PrimaryKey("PK_IdentityClaims", x => x.Id);
                    table.ForeignKey(
                        name: "FK_IdentityClaims_IdentityResources_IdentityResourceId",
                        column: x => x.IdentityResourceId,
                        principalTable: "IdentityResources",
                        principalColumn: "Id",
                        onDelete: ReferentialAction.Cascade);
                });

            migrationBuilder.CreateTable(
                name: "ApiScopeClaims",
                columns: table => new
                {
                    Id = table.Column<int>(type: "int", nullable: false)
                        .Annotation("SqlServer:ValueGenerationStrategy", SqlServerValueGenerationStrategy.IdentityColumn),
                    ApiScopeId = table.Column<int>(type: "int", nullable: false),
                    Type = table.Column<string>(type: "nvarchar(200)", maxLength: 200, nullable: false)
                },
                constraints: table =>
                {
                    table.PrimaryKey("PK_ApiScopeClaims", x => x.Id);
                    table.ForeignKey(
                        name: "FK_ApiScopeClaims_ApiScopes_ApiScopeId",
                        column: x => x.ApiScopeId,
                        principalTable: "ApiScopes",
                        principalColumn: "Id",
                        onDelete: ReferentialAction.Cascade);
                });

            migrationBuilder.CreateIndex(
                name: "IX_ApiClaims_ApiResourceId",
                table: "ApiClaims",
                column: "ApiResourceId");

            migrationBuilder.CreateIndex(
                name: "IX_ApiResources_Name",
                table: "ApiResources",
                column: "Name",
                unique: true);

            migrationBuilder.CreateIndex(
                name: "IX_ApiScopeClaims_ApiScopeId",
                table: "ApiScopeClaims",
                column: "ApiScopeId");

            migrationBuilder.CreateIndex(
                name: "IX_ApiScopes_ApiResourceId",
                table: "ApiScopes",
                column: "ApiResourceId");

            migrationBuilder.CreateIndex(
                name: "IX_ApiScopes_Name",
                table: "ApiScopes",
                column: "Name",
                unique: true);

            migrationBuilder.CreateIndex(
                name: "IX_ApiSecrets_ApiResourceId",
                table: "ApiSecrets",
                column: "ApiResourceId");

            migrationBuilder.CreateIndex(
                name: "IX_ClientClaims_ClientId",
                table: "ClientClaims",
                column: "ClientId");

            migrationBuilder.CreateIndex(
                name: "IX_ClientCorsOrigins_ClientId",
                table: "ClientCorsOrigins",
                column: "ClientId");

            migrationBuilder.CreateIndex(
                name: "IX_ClientGrantTypes_ClientId",
                table: "ClientGrantTypes",
                column: "ClientId");

            migrationBuilder.CreateIndex(
                name: "IX_ClientIdPRestrictions_ClientId",
                table: "ClientIdPRestrictions",
                column: "ClientId");

            migrationBuilder.CreateIndex(
                name: "IX_ClientPostLogoutRedirectUris_ClientId",
                table: "ClientPostLogoutRedirectUris",
                column: "ClientId");

            migrationBuilder.CreateIndex(
                name: "IX_ClientProperties_ClientId",
                table: "ClientProperties",
                column: "ClientId");

            migrationBuilder.CreateIndex(
                name: "IX_ClientRedirectUris_ClientId",
                table: "ClientRedirectUris",
                column: "ClientId");

            migrationBuilder.CreateIndex(
                name: "IX_Clients_ClientId",
                table: "Clients",
                column: "ClientId",
                unique: true);

            migrationBuilder.CreateIndex(
                name: "IX_ClientScopes_ClientId",
                table: "ClientScopes",
                column: "ClientId");

            migrationBuilder.CreateIndex(
                name: "IX_ClientSecrets_ClientId",
                table: "ClientSecrets",
                column: "ClientId");

            migrationBuilder.CreateIndex(
                name: "IX_IdentityClaims_IdentityResourceId",
                table: "IdentityClaims",
                column: "IdentityResourceId");

            migrationBuilder.CreateIndex(
                name: "IX_IdentityResources_Name",
                table: "IdentityResources",
                column: "Name",
                unique: true);
        }

        protected override void Down(MigrationBuilder migrationBuilder)
        {
            migrationBuilder.DropTable(
                name: "ApiClaims");

            migrationBuilder.DropTable(
                name: "ApiScopeClaims");

            migrationBuilder.DropTable(
                name: "ApiSecrets");

            migrationBuilder.DropTable(
                name: "ClientClaims");

            migrationBuilder.DropTable(
                name: "ClientCorsOrigins");

            migrationBuilder.DropTable(
                name: "ClientGrantTypes");

            migrationBuilder.DropTable(
                name: "ClientIdPRestrictions");

            migrationBuilder.DropTable(
                name: "ClientPostLogoutRedirectUris");

            migrationBuilder.DropTable(
                name: "ClientProperties");

            migrationBuilder.DropTable(
                name: "ClientRedirectUris");

            migrationBuilder.DropTable(
                name: "ClientScopes");

            migrationBuilder.DropTable(
                name: "ClientSecrets");

            migrationBuilder.DropTable(
                name: "IdentityClaims");

            migrationBuilder.DropTable(
                name: "ApiScopes");

            migrationBuilder.DropTable(
                name: "Clients");

            migrationBuilder.DropTable(
                name: "IdentityResources");

            migrationBuilder.DropTable(
                name: "ApiResources");
        }
    }
}


text:
package com.thoughtworks.go.config;

import com.thoughtworks.go.config.policy.*;
import com.thoughtworks.go.config.validation.NameTypeValidator;
import com.thoughtworks.go.domain.ConfigErrors;

import java.util.*;

import static com.thoughtworks.go.config.CaseInsensitiveString.isBlank;
import static com.thoughtworks.go.config.policy.SupportedAction.ADMINISTER;
import static com.thoughtworks.go.config.policy.SupportedAction.VIEW;
import static com.thoughtworks.go.config.policy.SupportedEntity.*;

@ConfigInterface
public interface Role extends Validatable, PolicyAware {
    List<String> allowedActions = SupportedAction.unmodifiableListOf(VIEW, ADMINISTER);
    List<String> allowedTypes = SupportedEntity.unmodifiableListOf(ENVIRONMENT, CONFIG_REPO, ELASTIC_AGENT_PROFILE, CLUSTER_PROFILE);

    CaseInsensitiveString getName();

    void setName(CaseInsensitiveString name);

    List<RoleUser> getUsers();

    void addUser(RoleUser user);

    void removeUser(RoleUser roleUser);

    default boolean validateTree(ValidationContext validationContext) {
        validate(validationContext);
        getPolicy().validateTree(new DelegatingValidationContext(validationContext) {
            @Override
            public PolicyValidationContext getPolicyValidationContext() {
                return new PolicyValidationContext(allowedActions(), allowedTypes());
            }
        });
        return !hasErrors();
    }

    @Override
    default void validate(ValidationContext validationContext) {
        if (isBlank(getName()) || !new NameTypeValidator().isNameValid(getName())) {
            addError("name", NameTypeValidator.errorMessage("role name", getName()));
        }

        RolesConfig roles = validationContext.getServerSecurityConfig().getRoles();

        if (!isBlank(getName()) && !roles.isUniqueRoleName(getName())) {
            addError("name", "Role names should be unique. Role with the same name exists.");
        }

        Set<RoleUser> roleUsers = new HashSet<>();
        for (RoleUser user : getUsers()) {
            if (roleUsers.contains(user)) {
                new ErrorMarkingDuplicateHandler(this).invoke(user);
            } else {
                roleUsers.add(user);
            }
        }
    }

    default boolean hasMember(CaseInsensitiveString user) {
        if (user == null) {
            return false;
        }
        for (RoleUser roleUser : getUsers()) {
            if (roleUser.getName().equals(user)) {
                return true;
            }
        }
        return false;
    }

    default Collection<String> usersOfRole() {
        List<String> users = new ArrayList<>();
        for (RoleUser roleUser : getUsers()) {
            users.add(CaseInsensitiveString.str(roleUser.getName()));
        }
        return users;
    }

    default List<ConfigErrors> getAllErrors() {
        return ErrorCollector.getAllErrors(this);
    }

    boolean hasErrors();

    void encryptSecureProperties(CruiseConfig preprocessedConfig);

    class ErrorMarkingDuplicateHandler {

        private final Role role;

        public ErrorMarkingDuplicateHandler(Role role) {
            this.role = role;
        }

        public void invoke(final RoleUser roleUser) {
            roleUser.addDuplicateError(CaseInsensitiveString.str(role.getName()));
        }
    }

    Policy getPolicy();

    void setPolicy(Policy policy);

    @Override
    default List<String> allowedActions() {
        return allowedActions;
    }

    @Override
    default List<String> allowedTypes() {
        return allowedTypes;
    }
}


text:
using System;

namespace ComponentSystem.Exceptions
{
    /// <summary>This exception is thrown when a class was decorated with the ContentLoaderAttribute without deriving from a valid base class.</summary>
    public class InvalidContentLoaderBaseTypeException : Exception
    {
        public InvalidContentLoaderBaseTypeException(Type type, Type contentLoaderType) : base("The content loader '" + type.FullName + "' does not inherit from " + contentLoaderType.Name + ".")
        {

        }

        public InvalidContentLoaderBaseTypeException(string message, Exception innerException) : base(message, innerException)
        {

        }
    }
}


text:
package org.apache.hadoop.fs.azurebfs.services;

import java.io.IOException;

import org.junit.Assert;
import org.junit.Test;
import java.util.Arrays;

import org.assertj.core.api.Assertions;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.azurebfs.AbstractAbfsIntegrationTest;
import org.apache.hadoop.fs.azurebfs.AzureBlobFileSystem;
import org.apache.hadoop.fs.azurebfs.contracts.exceptions.AzureBlobFileSystemException;
import org.apache.hadoop.fs.azurebfs.contracts.exceptions.TimeoutException;
import org.apache.hadoop.fs.azurebfs.contracts.services.ReadBufferStatus;
import org.apache.hadoop.fs.azurebfs.utils.TestCachedSASToken;
import org.apache.hadoop.fs.azurebfs.utils.TracingContext;

import static org.mockito.ArgumentMatchers.any;
import static org.mockito.Mockito.doReturn;
import static org.mockito.Mockito.doThrow;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;

import static org.apache.hadoop.test.LambdaTestUtils.intercept;
import static org.apache.hadoop.fs.azurebfs.constants.AbfsHttpConstants.FORWARD_SLASH;
import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.FS_AZURE_READ_AHEAD_QUEUE_DEPTH;

/**
 * Unit test AbfsInputStream.
 */
public class TestAbfsInputStream extends
    AbstractAbfsIntegrationTest {

  private static final int ONE_KB = 1 * 1024;
  private static final int TWO_KB = 2 * 1024;
  private static final int THREE_KB = 3 * 1024;
  private static final int SIXTEEN_KB = 16 * ONE_KB;
  private static final int FORTY_EIGHT_KB = 48 * ONE_KB;
  private static final int ONE_MB = 1 * 1024 * 1024;
  private static final int FOUR_MB = 4 * ONE_MB;
  private static final int EIGHT_MB = 8 * ONE_MB;
  private static final int TEST_READAHEAD_DEPTH_2 = 2;
  private static final int TEST_READAHEAD_DEPTH_4 = 4;
  private static final int REDUCED_READ_BUFFER_AGE_THRESHOLD = 3000; // 3 sec
  private static final int INCREASED_READ_BUFFER_AGE_THRESHOLD =
      REDUCED_READ_BUFFER_AGE_THRESHOLD * 10; // 30 sec
  private static final int ALWAYS_READ_BUFFER_SIZE_TEST_FILE_SIZE = 16 * ONE_MB;

  private AbfsRestOperation getMockRestOp() {
    AbfsRestOperation op = mock(AbfsRestOperation.class);
    AbfsHttpOperation httpOp = mock(AbfsHttpOperation.class);
    when(httpOp.getBytesReceived()).thenReturn(1024L);
    when(op.getResult()).thenReturn(httpOp);
    when(op.getSasToken()).thenReturn(TestCachedSASToken.getTestCachedSASTokenInstance().get());
    return op;
  }

  private AbfsClient getMockAbfsClient() {
    // Mock failure for client.read()
    AbfsClient client = mock(AbfsClient.class);
    AbfsPerfTracker tracker = new AbfsPerfTracker(
        "test",
        this.getAccountName(),
        this.getConfiguration());
    when(client.getAbfsPerfTracker()).thenReturn(tracker);

    return client;
  }

  private AbfsInputStream getAbfsInputStream(AbfsClient mockAbfsClient,
      String fileName) throws IOException {
    AbfsInputStreamContext inputStreamContext = new AbfsInputStreamContext(-1);
    // Create AbfsInputStream with the client instance
    AbfsInputStream inputStream = new AbfsInputStream(
        mockAbfsClient,
        null,
        FORWARD_SLASH + fileName,
        THREE_KB,
        inputStreamContext.withReadBufferSize(ONE_KB).withReadAheadQueueDepth(10).withReadAheadBlockSize(ONE_KB),
        "eTag",
        getTestTracingContext(null, false));

    inputStream.setCachedSasToken(
        TestCachedSASToken.getTestCachedSASTokenInstance());

    return inputStream;
  }

  public AbfsInputStream getAbfsInputStream(AbfsClient abfsClient,
      String fileName,
      int fileSize,
      String eTag,
      int readAheadQueueDepth,
      int readBufferSize,
      boolean alwaysReadBufferSize,
      int readAheadBlockSize) throws IOException {
    AbfsInputStreamContext inputStreamContext = new AbfsInputStreamContext(-1);
    // Create AbfsInputStream with the client instance
    AbfsInputStream inputStream = new AbfsInputStream(
        abfsClient,
        null,
        FORWARD_SLASH + fileName,
        fileSize,
        inputStreamContext.withReadBufferSize(readBufferSize)
            .withReadAheadQueueDepth(readAheadQueueDepth)
            .withShouldReadBufferSizeAlways(alwaysReadBufferSize)
            .withReadAheadBlockSize(readAheadBlockSize),
        eTag,
        getTestTracingContext(getFileSystem(), false));

    inputStream.setCachedSasToken(
        TestCachedSASToken.getTestCachedSASTokenInstance());

    return inputStream;
  }

  private void queueReadAheads(AbfsInputStream inputStream) {
    // Mimic AbfsInputStream readAhead queue requests
    ReadBufferManager.getBufferManager()
        .queueReadAhead(inputStream, 0, ONE_KB, inputStream.getTracingContext());
    ReadBufferManager.getBufferManager()
        .queueReadAhead(inputStream, ONE_KB, ONE_KB,
            inputStream.getTracingContext());
    ReadBufferManager.getBufferManager()
        .queueReadAhead(inputStream, TWO_KB, TWO_KB,
            inputStream.getTracingContext());
  }

  private void verifyReadCallCount(AbfsClient client, int count) throws
      AzureBlobFileSystemException, InterruptedException {
    // ReadAhead threads are triggered asynchronously.
    // Wait a second before verifying the number of total calls.
    Thread.sleep(1000);
    verify(client, times(count)).read(any(String.class), any(Long.class),
        any(byte[].class), any(Integer.class), any(Integer.class),
        any(String.class), any(String.class), any(TracingContext.class));
  }

  private void checkEvictedStatus(AbfsInputStream inputStream, int position, boolean expectedToThrowException)
      throws Exception {
    // Sleep for the eviction threshold time
    Thread.sleep(ReadBufferManager.getBufferManager().getThresholdAgeMilliseconds() + 1000);

    // Eviction is done only when AbfsInputStream tries to queue new items.
    // 1 tryEvict will remove 1 eligible item. To ensure that the current test buffer
    // will get evicted (considering there could be other tests running in parallel),
    // call tryEvict for the number of items that are there in completedReadList.
    int numOfCompletedReadListItems = ReadBufferManager.getBufferManager().getCompletedReadListSize();
    while (numOfCompletedReadListItems > 0) {
      ReadBufferManager.getBufferManager().callTryEvict();
      numOfCompletedReadListItems--;
    }

    if (expectedToThrowException) {
      intercept(IOException.class,
          () -> inputStream.read(position, new byte[ONE_KB], 0, ONE_KB));
    } else {
      inputStream.read(position, new byte[ONE_KB], 0, ONE_KB);
    }
  }

  public TestAbfsInputStream() throws Exception {
    super();
    // Reduce thresholdAgeMilliseconds to 3 sec for the tests
    ReadBufferManager.getBufferManager().setThresholdAgeMilliseconds(REDUCED_READ_BUFFER_AGE_THRESHOLD);
  }

  /**
   * This test expects AbfsInputStream to throw the exception that readAhead
   * thread received on read. The readAhead thread must be initiated from the
   * active read request itself.
   * Also checks that the ReadBuffers are evicted as per the ReadBufferManager
   * threshold criteria.
   * @throws Exception
   */
  @Test
  public void testFailedReadAhead() throws Exception {
    AbfsClient client = getMockAbfsClient();
    AbfsRestOperation successOp = getMockRestOp();

    // Stub :
    // Read request leads to 3 readahead calls: Fail all 3 readahead-client.read()
    // Actual read request fails with the failure in readahead thread
    doThrow(new TimeoutException("Internal Server error for RAH-Thread-X"))
        .doThrow(new TimeoutException("Internal Server error for RAH-Thread-Y"))
        .doThrow(new TimeoutException("Internal Server error RAH-Thread-Z"))
        .doReturn(successOp) // Any extra calls to read, pass it.
        .when(client)
        .read(any(String.class), any(Long.class), any(byte[].class),
            any(Integer.class), any(Integer.class), any(String.class),
            any(String.class), any(TracingContext.class));

    AbfsInputStream inputStream = getAbfsInputStream(client, "testFailedReadAhead.txt");

    // Scenario: ReadAhead triggered from current active read call failed
    // Before the change to return exception from readahead buffer,
    // AbfsInputStream would have triggered an extra readremote on noticing
    // data absent in readahead buffers
    // In this test, a read should trigger 3 client.read() calls as file is 3 KB
    // and readahead buffer size set in AbfsInputStream is 1 KB
    // There should only be a total of 3 client.read() in this test.
    intercept(IOException.class,
        () -> inputStream.read(new byte[ONE_KB]));

    // Only the 3 readAhead threads should have triggered client.read
    verifyReadCallCount(client, 3);

    // Stub returns success for the 4th read request, if ReadBuffers still
    // persisted, ReadAheadManager getBlock would have returned exception.
    checkEvictedStatus(inputStream, 0, false);
  }

  @Test
  public void testFailedReadAheadEviction() throws Exception {
    AbfsClient client = getMockAbfsClient();
    AbfsRestOperation successOp = getMockRestOp();
    ReadBufferManager.setThresholdAgeMilliseconds(INCREASED_READ_BUFFER_AGE_THRESHOLD);
    // Stub :
    // Read request leads to 3 readahead calls: Fail all 3 readahead-client.read()
    // Actual read request fails with the failure in readahead thread
    doThrow(new TimeoutException("Internal Server error"))
        .when(client)
        .read(any(String.class), any(Long.class), any(byte[].class),
            any(Integer.class), any(Integer.class), any(String.class),
            any(String.class), any(TracingContext.class));

    AbfsInputStream inputStream = getAbfsInputStream(client, "testFailedReadAheadEviction.txt");

    // Add a failed buffer to completed queue and set to no free buffers to read ahead.
    ReadBuffer buff = new ReadBuffer();
    buff.setStatus(ReadBufferStatus.READ_FAILED);
    ReadBufferManager.getBufferManager().testMimicFullUseAndAddFailedBuffer(buff);

    // if read failed buffer eviction is tagged as a valid eviction, it will lead to
    // wrong assumption of queue logic that a buffer is freed up and can lead to :
    // java.util.EmptyStackException
    // at java.util.Stack.peek(Stack.java:102)
    // at java.util.Stack.pop(Stack.java:84)
    // at org.apache.hadoop.fs.azurebfs.services.ReadBufferManager.queueReadAhead
    ReadBufferManager.getBufferManager().queueReadAhead(inputStream, 0, ONE_KB,
        getTestTracingContext(getFileSystem(), true));
  }

  /**
   *
   * The test expects AbfsInputStream to initiate a remote read request for
   * the request offset and length when previous read ahead on the offset had failed.
   * Also checks that the ReadBuffers are evicted as per the ReadBufferManager
   * threshold criteria.
   * @throws Exception
   */
  @Test
  public void testOlderReadAheadFailure() throws Exception {
    AbfsClient client = getMockAbfsClient();
    AbfsRestOperation successOp = getMockRestOp();

    // Stub :
    // First Read request leads to 3 readahead calls: Fail all 3 readahead-client.read()
    // A second read request will see that readahead had failed for data in
    // the requested offset range and also that its is an older readahead request.
    // So attempt a new read only for the requested range.
    doThrow(new TimeoutException("Internal Server error for RAH-X"))
        .doThrow(new TimeoutException("Internal Server error for RAH-Y"))
        .doThrow(new TimeoutException("Internal Server error for RAH-Z"))
        .doReturn(successOp) // pass the read for second read request
        .doReturn(successOp) // pass success for post eviction test
        .when(client)
        .read(any(String.class), any(Long.class), any(byte[].class),
            any(Integer.class), any(Integer.class), any(String.class),
            any(String.class), any(TracingContext.class));

    AbfsInputStream inputStream = getAbfsInputStream(client, "testOlderReadAheadFailure.txt");

    // First read request that fails as the readahead triggered from this request failed.
    intercept(IOException.class,
        () -> inputStream.read(new byte[ONE_KB]));

    // Only the 3 readAhead threads should have triggered client.read
    verifyReadCallCount(client, 3);

    // Sleep for thresholdAgeMs so that the read ahead buffer qualifies for being old.
    Thread.sleep(ReadBufferManager.getBufferManager().getThresholdAgeMilliseconds());

    // Second read request should retry the read (and not issue any new readaheads)
    inputStream.read(ONE_KB, new byte[ONE_KB], 0, ONE_KB);

    // Once created, mock will remember all interactions. So total number of read
    // calls will be one more from earlier (there is a reset mock which will reset the
    // count, but the mock stub is erased as well which needs AbsInputStream to be recreated,
    // which beats the purpose)
    verifyReadCallCount(client, 4);

    // Stub returns success for the 5th read request, if ReadBuffers still
    // persisted request would have failed for position 0.
    checkEvictedStatus(inputStream, 0, false);
  }

  /**
   * The test expects AbfsInputStream to utilize any data read ahead for
   * requested offset and length.
   * @throws Exception
   */
  @Test
  public void testSuccessfulReadAhead() throws Exception {
    // Mock failure for client.read()
    AbfsClient client = getMockAbfsClient();

    // Success operation mock
    AbfsRestOperation op = getMockRestOp();

    // Stub :
    // Pass all readAheads and fail the post eviction request to
    // prove ReadAhead buffer is used
    // for post eviction check, fail all read aheads
    doReturn(op)
        .doReturn(op)
        .doReturn(op)
        .doThrow(new TimeoutException("Internal Server error for RAH-X"))
        .doThrow(new TimeoutException("Internal Server error for RAH-Y"))
        .doThrow(new TimeoutException("Internal Server error for RAH-Z"))
        .when(client)
        .read(any(String.class), any(Long.class), any(byte[].class),
            any(Integer.class), any(Integer.class), any(String.class),
            any(String.class), any(TracingContext.class));

    AbfsInputStream inputStream = getAbfsInputStream(client, "testSuccessfulReadAhead.txt");
    int beforeReadCompletedListSize = ReadBufferManager.getBufferManager().getCompletedReadListSize();

    // First read request that triggers readAheads.
    inputStream.read(new byte[ONE_KB]);

    // Only the 3 readAhead threads should have triggered client.read
    verifyReadCallCount(client, 3);
    int newAdditionsToCompletedRead =
        ReadBufferManager.getBufferManager().getCompletedReadListSize()
            - beforeReadCompletedListSize;
    // read buffer might be dumped if the ReadBufferManager getblock preceded
    // the action of buffer being picked for reading from readaheadqueue, so that
    // inputstream can proceed with read and not be blocked on readahead thread
    // availability. So the count of buffers in completedReadQueue for the stream
    // can be same or lesser than the requests triggered to queue readahead.
    Assertions.assertThat(newAdditionsToCompletedRead)
        .describedAs(
            "New additions to completed reads should be same or less than as number of readaheads")
        .isLessThanOrEqualTo(3);

    // Another read request whose requested data is already read ahead.
    inputStream.read(ONE_KB, new byte[ONE_KB], 0, ONE_KB);

    // Once created, mock will remember all interactions.
    // As the above read should not have triggered any server calls, total
    // number of read calls made at this point will be same as last.
    verifyReadCallCount(client, 3);

    // Stub will throw exception for client.read() for 4th and later calls
    // if not using the read-ahead buffer exception will be thrown on read
    checkEvictedStatus(inputStream, 0, true);
  }

  /**
   * This test expects ReadAheadManager to throw exception if the read ahead
   * thread had failed within the last thresholdAgeMilliseconds.
   * Also checks that the ReadBuffers are evicted as per the ReadBufferManager
   * threshold criteria.
   * @throws Exception
   */
  @Test
  public void testReadAheadManagerForFailedReadAhead() throws Exception {
    AbfsClient client = getMockAbfsClient();
    AbfsRestOperation successOp = getMockRestOp();

    // Stub :
    // Read request leads to 3 readahead calls: Fail all 3 readahead-client.read()
    // Actual read request fails with the failure in readahead thread
    doThrow(new TimeoutException("Internal Server error for RAH-Thread-X"))
        .doThrow(new TimeoutException("Internal Server error for RAH-Thread-Y"))
        .doThrow(new TimeoutException("Internal Server error RAH-Thread-Z"))
        .doReturn(successOp) // Any extra calls to read, pass it.
        .when(client)
        .read(any(String.class), any(Long.class), any(byte[].class),
            any(Integer.class), any(Integer.class), any(String.class),
            any(String.class), any(TracingContext.class));

    AbfsInputStream inputStream = getAbfsInputStream(client, "testReadAheadManagerForFailedReadAhead.txt");

    queueReadAheads(inputStream);

    // AbfsInputStream Read would have waited for the read-ahead for the requested offset
    // as we are testing from ReadAheadManager directly, sleep for a sec to
    // get the read ahead threads to complete
    Thread.sleep(1000);

    // if readAhead failed for specific offset, getBlock should
    // throw exception from the ReadBuffer that failed within last thresholdAgeMilliseconds sec
    intercept(IOException.class,
        () -> ReadBufferManager.getBufferManager().getBlock(
            inputStream,
            0,
            ONE_KB,
            new byte[ONE_KB]));

    // Only the 3 readAhead threads should have triggered client.read
    verifyReadCallCount(client, 3);

    // Stub returns success for the 4th read request, if ReadBuffers still
    // persisted, ReadAheadManager getBlock would have returned exception.
    checkEvictedStatus(inputStream, 0, false);
  }

  /**
   * The test expects ReadAheadManager to return 0 receivedBytes when previous
   * read ahead on the offset had failed and not throw exception received then.
   * Also checks that the ReadBuffers are evicted as per the ReadBufferManager
   * threshold criteria.
   * @throws Exception
   */
  @Test
  public void testReadAheadManagerForOlderReadAheadFailure() throws Exception {
    AbfsClient client = getMockAbfsClient();
    AbfsRestOperation successOp = getMockRestOp();

    // Stub :
    // First Read request leads to 3 readahead calls: Fail all 3 readahead-client.read()
    // A second read request will see that readahead had failed for data in
    // the requested offset range but also that its is an older readahead request.
    // System issue could have resolved by now, so attempt a new read only for the requested range.
    doThrow(new TimeoutException("Internal Server error for RAH-X"))
        .doThrow(new TimeoutException("Internal Server error for RAH-X"))
        .doThrow(new TimeoutException("Internal Server error for RAH-X"))
        .doReturn(successOp) // pass the read for second read request
        .doReturn(successOp) // pass success for post eviction test
        .when(client)
        .read(any(String.class), any(Long.class), any(byte[].class),
            any(Integer.class), any(Integer.class), any(String.class),
            any(String.class), any(TracingContext.class));

    AbfsInputStream inputStream = getAbfsInputStream(client, "testReadAheadManagerForOlderReadAheadFailure.txt");

    queueReadAheads(inputStream);

    // AbfsInputStream Read would have waited for the read-ahead for the requested offset
    // as we are testing from ReadAheadManager directly, sleep for thresholdAgeMilliseconds so that
    // read buffer qualifies for to be an old buffer
    Thread.sleep(ReadBufferManager.getBufferManager().getThresholdAgeMilliseconds());

    // Only the 3 readAhead threads should have triggered client.read
    verifyReadCallCount(client, 3);

    // getBlock from a new read request should return 0 if there is a failure
    // 30 sec before in read ahead buffer for respective offset.
    int bytesRead = ReadBufferManager.getBufferManager().getBlock(
        inputStream,
        ONE_KB,
        ONE_KB,
        new byte[ONE_KB]);
    Assert.assertEquals("bytesRead should be zero when previously read "
        + "ahead buffer had failed", 0, bytesRead);

    // Stub returns success for the 5th read request, if ReadBuffers still
    // persisted request would have failed for position 0.
    checkEvictedStatus(inputStream, 0, false);
  }

  /**
   * The test expects ReadAheadManager to return data from previously read
   * ahead data of same offset.
   * @throws Exception
   */
  @Test
  public void testReadAheadManagerForSuccessfulReadAhead() throws Exception {
    // Mock failure for client.read()
    AbfsClient client = getMockAbfsClient();

    // Success operation mock
    AbfsRestOperation op = getMockRestOp();

    // Stub :
    // Pass all readAheads and fail the post eviction request to
    // prove ReadAhead buffer is used
    doReturn(op)
        .doReturn(op)
        .doReturn(op)
        .doThrow(new TimeoutException("Internal Server error for RAH-X")) // for post eviction request
        .doThrow(new TimeoutException("Internal Server error for RAH-Y"))
        .doThrow(new TimeoutException("Internal Server error for RAH-Z"))
        .when(client)
        .read(any(String.class), any(Long.class), any(byte[].class),
            any(Integer.class), any(Integer.class), any(String.class),
            any(String.class), any(TracingContext.class));

    AbfsInputStream inputStream = getAbfsInputStream(client, "testSuccessfulReadAhead.txt");

    queueReadAheads(inputStream);

    // AbfsInputStream Read would have waited for the read-ahead for the requested offset
    // as we are testing from ReadAheadManager directly, sleep for a sec to
    // get the read ahead threads to complete
    Thread.sleep(1000);

    // Only the 3 readAhead threads should have triggered client.read
    verifyReadCallCount(client, 3);

    // getBlock for a new read should return the buffer read-ahead
    int bytesRead = ReadBufferManager.getBufferManager().getBlock(
        inputStream,
        ONE_KB,
        ONE_KB,
        new byte[ONE_KB]);

    Assert.assertTrue("bytesRead should be non-zero from the "
        + "buffer that was read-ahead", bytesRead > 0);

    // Once created, mock will remember all interactions.
    // As the above read should not have triggered any server calls, total
    // number of read calls made at this point will be same as last.
    verifyReadCallCount(client, 3);

    // Stub will throw exception for client.read() for 4th and later calls
    // if not using the read-ahead buffer exception will be thrown on read
    checkEvictedStatus(inputStream, 0, true);
  }

  /**
   * Test readahead with different config settings for request request size and
   * readAhead block size
   * @throws Exception
   */
  @Test
  public void testDiffReadRequestSizeAndRAHBlockSize() throws Exception {
    // Set requestRequestSize = 4MB and readAheadBufferSize=8MB
    resetReadBufferManager(FOUR_MB, INCREASED_READ_BUFFER_AGE_THRESHOLD);
    testReadAheadConfigs(FOUR_MB, TEST_READAHEAD_DEPTH_4, false, EIGHT_MB);

    // Test for requestRequestSize =16KB and readAheadBufferSize=16KB
    resetReadBufferManager(SIXTEEN_KB, INCREASED_READ_BUFFER_AGE_THRESHOLD);
    AbfsInputStream inputStream = testReadAheadConfigs(SIXTEEN_KB,
        TEST_READAHEAD_DEPTH_2, true, SIXTEEN_KB);
    testReadAheads(inputStream, SIXTEEN_KB, SIXTEEN_KB);

    // Test for requestRequestSize =16KB and readAheadBufferSize=48KB
    resetReadBufferManager(FORTY_EIGHT_KB, INCREASED_READ_BUFFER_AGE_THRESHOLD);
    inputStream = testReadAheadConfigs(SIXTEEN_KB, TEST_READAHEAD_DEPTH_2, true,
        FORTY_EIGHT_KB);
    testReadAheads(inputStream, SIXTEEN_KB, FORTY_EIGHT_KB);

    // Test for requestRequestSize =48KB and readAheadBufferSize=16KB
    resetReadBufferManager(FORTY_EIGHT_KB, INCREASED_READ_BUFFER_AGE_THRESHOLD);
    inputStream = testReadAheadConfigs(FORTY_EIGHT_KB, TEST_READAHEAD_DEPTH_2,
        true,
        SIXTEEN_KB);
    testReadAheads(inputStream, FORTY_EIGHT_KB, SIXTEEN_KB);
  }

  @Test
  public void testDefaultReadaheadQueueDepth() throws Exception {
    Configuration config = getRawConfiguration();
    config.unset(FS_AZURE_READ_AHEAD_QUEUE_DEPTH);
    AzureBlobFileSystem fs = getFileSystem(config);
    Path testFile = path("/testFile");
    fs.create(testFile).close();
    FSDataInputStream in = fs.open(testFile);
    Assertions.assertThat(
        ((AbfsInputStream) in.getWrappedStream()).getReadAheadQueueDepth())
        .describedAs("readahead queue depth should be set to default value 2")
        .isEqualTo(2);
    in.close();
  }

  private void testReadAheads(AbfsInputStream inputStream,
      int readRequestSize,
      int readAheadRequestSize)
      throws Exception {
    if (readRequestSize > readAheadRequestSize) {
      readAheadRequestSize = readRequestSize;
    }

    byte[] firstReadBuffer = new byte[readRequestSize];
    byte[] secondReadBuffer = new byte[readAheadRequestSize];

    // get the expected bytes to compare
    byte[] expectedFirstReadAheadBufferContents = new byte[readRequestSize];
    byte[] expectedSecondReadAheadBufferContents = new byte[readAheadRequestSize];
    getExpectedBufferData(0, readRequestSize, expectedFirstReadAheadBufferContents);
    getExpectedBufferData(readRequestSize, readAheadRequestSize,
        expectedSecondReadAheadBufferContents);

    Assertions.assertThat(inputStream.read(firstReadBuffer, 0, readRequestSize))
        .describedAs("Read should be of exact requested size")
        .isEqualTo(readRequestSize);

    assertTrue("Data mismatch found in RAH1",
        Arrays.equals(firstReadBuffer,
            expectedFirstReadAheadBufferContents));

    Assertions.assertThat(inputStream.read(secondReadBuffer, 0, readAheadRequestSize))
        .describedAs("Read should be of exact requested size")
        .isEqualTo(readAheadRequestSize);

    assertTrue("Data mismatch found in RAH2",
        Arrays.equals(secondReadBuffer,
            expectedSecondReadAheadBufferContents));
  }

  public AbfsInputStream testReadAheadConfigs(int readRequestSize,
      int readAheadQueueDepth,
      boolean alwaysReadBufferSizeEnabled,
      int readAheadBlockSize) throws Exception {
    Configuration
        config = new Configuration(
        this.getRawConfiguration());
    config.set("fs.azure.read.request.size", Integer.toString(readRequestSize));
    config.set("fs.azure.readaheadqueue.depth",
        Integer.toString(readAheadQueueDepth));
    config.set("fs.azure.read.alwaysReadBufferSize",
        Boolean.toString(alwaysReadBufferSizeEnabled));
    config.set("fs.azure.read.readahead.blocksize",
        Integer.toString(readAheadBlockSize));
    if (readRequestSize > readAheadBlockSize) {
      readAheadBlockSize = readRequestSize;
    }

    Path testPath = path("/testReadAheadConfigs");
    final AzureBlobFileSystem fs = createTestFile(testPath,
        ALWAYS_READ_BUFFER_SIZE_TEST_FILE_SIZE, config);
    byte[] byteBuffer = new byte[ONE_MB];
    AbfsInputStream inputStream = this.getAbfsStore(fs)
        .openFileForRead(testPath, null, getTestTracingContext(fs, false));

    Assertions.assertThat(inputStream.getBufferSize())
        .describedAs("Unexpected AbfsInputStream buffer size")
        .isEqualTo(readRequestSize);

    Assertions.assertThat(inputStream.getReadAheadQueueDepth())
        .describedAs("Unexpected ReadAhead queue depth")
        .isEqualTo(readAheadQueueDepth);

    Assertions.assertThat(inputStream.shouldAlwaysReadBufferSize())
        .describedAs("Unexpected AlwaysReadBufferSize settings")
        .isEqualTo(alwaysReadBufferSizeEnabled);

    Assertions.assertThat(ReadBufferManager.getBufferManager().getReadAheadBlockSize())
        .describedAs("Unexpected readAhead block size")
        .isEqualTo(readAheadBlockSize);

    return inputStream;
  }

  private void getExpectedBufferData(int offset, int length, byte[] b) {
    boolean startFillingIn = false;
    int indexIntoBuffer = 0;
    char character = 'a';

    for (int i = 0; i < (offset + length); i++) {
      if (i == offset) {
        startFillingIn = true;
      }

      if ((startFillingIn) && (indexIntoBuffer < length)) {
        b[indexIntoBuffer] = (byte) character;
        indexIntoBuffer++;
      }

      character = (character == 'z') ? 'a' : (char) ((int) character + 1);
    }
  }

  private AzureBlobFileSystem createTestFile(Path testFilePath, long testFileSize,
      Configuration config) throws Exception {
    AzureBlobFileSystem fs;

    if (config == null) {
      fs = this.getFileSystem();
    } else {
      final AzureBlobFileSystem currentFs = getFileSystem();
      fs = (AzureBlobFileSystem) FileSystem.newInstance(currentFs.getUri(),
          config);
    }

    if (fs.exists(testFilePath)) {
      FileStatus status = fs.getFileStatus(testFilePath);
      if (status.getLen() >= testFileSize) {
        return fs;
      }
    }

    byte[] buffer = new byte[EIGHT_MB];
    char character = 'a';
    for (int i = 0; i < buffer.length; i++) {
      buffer[i] = (byte) character;
      character = (character == 'z') ? 'a' : (char) ((int) character + 1);
    }

    try (FSDataOutputStream outputStream = fs.create(testFilePath)) {
      int bytesWritten = 0;
      while (bytesWritten < testFileSize) {
        outputStream.write(buffer);
        bytesWritten += buffer.length;
      }
    }

    Assertions.assertThat(fs.getFileStatus(testFilePath).getLen())
        .describedAs("File not created of expected size")
        .isEqualTo(testFileSize);

    return fs;
  }

  private void resetReadBufferManager(int bufferSize, int threshold) {
    ReadBufferManager.getBufferManager()
        .testResetReadBufferManager(bufferSize, threshold);
    // Trigger GC as aggressive recreation of ReadBufferManager buffers
    // by successive tests can lead to OOM based on the dev VM/machine capacity.
    System.gc();
  }
}


text:
{% extends "main-no-disqus.html" %}

{% block htmltitle %}
  <title>Search | {{ config.site_name }}</title>
{% endblock %}

{% block breadcrumbs %}

» Search

{% endblock %}

{% block extrahead %}

{% endblock %}

{% block content %}

Search Results

  <form id="content_search" action="search.html">

    <input name="q" id="mkdocs-search-query" type="text" class="search_input search-query ui-autocomplete-input" placeholder="Search the Docs" autocomplete="off" autofocus>
  </form>

Searching...

{% endblock %}


text:
package org.apache.pig.parser;

import java.io.File;
import java.io.IOException;

import junit.framework.Assert;

import org.antlr.runtime.MismatchedTokenException;
import org.antlr.runtime.NoViableAltException;
import org.apache.pig.test.Util;
import org.junit.BeforeClass;
import org.junit.Test;

public class TestLogicalPlanGenerator {
    static File command;

    @BeforeClass
    public static void oneTimeSetup() throws IOException, Exception {
        // Perl script
        String[] script =
            new String[] {
                          "#!/usr/bin/perl",
                          "open(INFILE,  $ARGV[0]) or die \"Can't open \".$ARGV[0].\"!: $!\";",
                          "while (<INFILE>) {",
                          "  chomp $_;",
                          "  print STDOUT \"$_\n\";",
                          "  print STDERR \"STDERR: $_\n\";",
                          "}",
                         };
        command = Util.createInputFile("script", "pl", script);
    }

    @Test
    public void test1() {
        String query = "A = load 'x' as ( u:int, v:long, w:bytearray); " +
                       "B = limit A 100; " +
                       "C = filter B by 2 > 1; " +
                       "D = load 'y' as (d1, d2); " +
                       "E = join C by ( $0, $1 ), D by ( d1, d2 ) using 'replicated' parallel 16; " +
                       "F = store E into 'output';";
        generateLogicalPlan( query );

    }

    @Test
    public void test2() {
        String query = "A = load 'x' as ( u:int, v:long, w:bytearray); " +
                       "B = distinct A partition by org.apache.pig.Identity; " +
                       "C = sample B 0.49; " +
                       "D = order C by $0, $1; " +
                       "E = load 'y' as (d1, d2); " +
                       "F = union onschema D, E; " +
                       "G = load 'z' as (g1:int, g2:tuple(g21, g22)); " +
                       "H = cross F, G; " +
                       "split H into I if 10 > 5, J if 'world' eq 'hello', K if 77 <= 200; " +
                       "L = store J into 'output';";
        generateLogicalPlan( query );
    }

    @Test
    public void test3() {
        String query = "a = load '1.txt'  as (name, age, gpa);" +
                       "b = group a by name PARTITION BY org.apache.pig.test.utils.SimpleCustomPartitioner2;" +
                       "c = foreach b generate group, COUNT(a.age);" +
                       "store c into 'y';";
        generateLogicalPlan( query );
    }

    private void generateLogicalPlan(String query) {
        try {
            ParserTestingUtils.generateLogicalPlan( query );
        } catch(Exception ex) {
            Assert.fail( "Failed to generate logical plan for query [" + query + "] due to exception: " + ex );
        }
    }

    @Test
    public void test4() {
        String query = "A = load 'x'; " +
                       "B = mapreduce '" + "myjar.jar" + "' " +
                           "Store A into 'table_testNativeMRJobSimple_input' "+
                           "Load 'table_testNativeMRJobSimple_output' "+
                           "`org.apache.pig.test.utils.WordCount -files " + "file " +
                           "table_testNativeMRJobSimple_input table_testNativeMRJobSimple_output " +
                           "stopworld.file" + "`;" +
                        "C = Store B into 'output';";
        generateLogicalPlan( query );
    }

    // Test define function.
    @Test
    public void test5() {
        String query = "define myudf org.apache.pig.builtin.PigStorage( ',' );" +
                       "A = load 'x' using myudf;" +
                       "store A into 'y';";
        generateLogicalPlan( query );
    }

    @Test
    public void test6() {
        String query = "A = load 'x' as ( a : int, b, c : chararray );" +
                       "B = group A by ( a, $2 );" +
                       "store B into 'y';";
        generateLogicalPlan( query );
    }

    @Test
    public void test7() {
        String query = "A = load 'x' as ( a : int, b, c : chararray );" +
                       "B = foreach A generate a, $2;" +
                       "store B into 'y';";
        generateLogicalPlan( query );
    }

    @Test
    public void test8() {
        String query = "A = load 'x' as ( a : int, b, c : chararray );" +
                       "B = group A by a;" +
                       "C = foreach B { S = A.b; generate S; };" +
                       "store C into 'y';";
        generateLogicalPlan( query );
    }

    @Test
    public void test9() {
        String query = "A = load 'x' as ( a : bag{ T:tuple(u, v) }, c : int, d : long );" +
                       "B = foreach A { R = a; S = R.u; T = limit S 100; generate S, T, c + d/5; };" +
                       "store B into 'y';";
        generateLogicalPlan( query );
    }

    @Test
    public void test10() {
        String query = "A = load 'x' as ( a : bag{ T:tuple(u, v) }, c : int, d : long );" +
                       "B = foreach A { S = a; T = limit S 100; generate T; };" +
                       "store B into 'y';";
        generateLogicalPlan( query );
    }

    @Test
    public void test11() {
        String query = "A = load 'x' as ( a : bag{ T:tuple(u, v) }, c : int, d : long );" +
                       "B = foreach A { T = limit a 100; generate T; };" +
                       "store B into 'y';";
        generateLogicalPlan( query );
    }

    @Test
    public void test12() {
        String query = "define CMD `perl GroupBy.pl '\t' 0 1` ship('"+Util.encodeEscape(command.toString())+"');" +
                       "A = load 'x';" +
                       "B = group A by $0;" +
                       "C = foreach B {" +
                       "   D = order A by $1; " +
                       "   generate flatten(D);" +
                       "};" +
                       "E = stream C through CMD;" +
                       "store E into 'y';";
        generateLogicalPlan( query );
    }

    @Test
    public void test13() {
        String query = "define CMD `perl PigStreaming.pl` ship('"+Util.encodeEscape(command.toString())+"') stderr('CMD');" +
                       "A = load 'x';" +
                       "C = stream A through CMD;" +
                       "store C into 'y';";
        generateLogicalPlan( query );
    }

    @Test
    public void test14() {
        String query = "a = load 'x1' using PigStorage() as (name, age:int, gpa);" +
                       "b = load 'x2' as (name, age, registration, contributions);" +
                       "e = cogroup a by name, b by name parallel 8;" +
                       "f = foreach e generate group,  SUM(a.age) as s;" +
                       "g = filter f by s>0;" +
                       "store g into 'y';";
        generateLogicalPlan( query );
    }

    @Test
    public void test15() {
        String query = "a = load 'x1' using PigStorage() as (name, age, gpa);" +
                       "b = group a all;" +
                       "c = foreach b generate AVG(a.age) as avg; " +
                       "d = load 'x2' using PigStorage() as (name, age, registration, contributions);" +
                       "e = group d all;" +
                       "f = foreach e generate AVG(d.age) as avg;" +
                       "y = foreach a generate age/c.avg, age/f.avg;" +
                       "store y into 'y';";
        generateLogicalPlan( query );
    }

    @Test
    public void test16() {
        String query = "AA = load 'x';" +
                       "A = foreach (group (filter AA by $0 > 0) all) generate flatten($1);" +
                       "store A into 'y';";
        generateLogicalPlan( query );
    }

    @Test
    public void test17() {
        String query = "store ( load 'x' ) into 'y';";
        generateLogicalPlan( query );
    }

    @Test
    public void test18() {
        String query = "A = load 'x';\n" +
                       "C = group (foreach A generate $0 parallel 5) all;";
        generateLogicalPlan( query );
    }

    @Test
    public void test19() {
        String query = "A = load 'x' as (u:map[], v);\n" +
                       "B = foreach A { T = (chararray)u#'hello'#'world'; generate T; };";
        generateLogicalPlan( query );
    }

    @Test
    public void test20() {
        String query = "A = load 'x' using PigStorage() as (a:int,b:chararray);\n" +
                       "B = foreach A { C = TOMAP()#'key1'; generate C as C; };";
        generateLogicalPlan( query );
    }

    @Test
    public void test21() {
        String query = "A = load 'x' as (u, v);\n" +
                       "B = foreach A { S = u; T = org.apache.pig.builtin.TOMAP(); generate S, T;};";
        generateLogicalPlan( query );
    }

    @Test
    public void test22() {
        String query = "A = (load 'x' as (u, v));\n" +
                       "B = (group (foreach A generate $0 parallel 5) all);";
        generateLogicalPlan( query );
    }

    @Test
    public void test23() {
        String query = "a = (load 'x1' using PigStorage() as (name, age, gpa));" +
                       "b = (group a all);" +
                       "c = (foreach b generate AVG(a.age) as avg); " +
                       "d = (load 'x2' using PigStorage() as (name, age, registration, contributions));" +
                       "e = (group d all);" +
                       "f = (foreach e generate AVG(d.age) as avg);" +
                       "y = (foreach a generate age/c.avg, age/f.avg);" +
                       "store y into 'y';";
        generateLogicalPlan( query );
    }

    @Test
    public void test24() {
        String query = "a = (load 'x1' using PigStorage() as (name, age:int, gpa));" +
                       "b = (load 'x2' as (name, age, registration, contributions));" +
                       "e = (cogroup a by name, b by name parallel 8);" +
                       "f = (foreach e generate group,  SUM(a.age) as s);" +
                       "g = (filter f by s>0);" +
                       "(store g into 'y');";
        generateLogicalPlan( query );
    }

    @Test
    public void test25() {
        String query = "A = (load 'x' as ( u:int, v:long, w:bytearray)); " +
                       "B = (distinct A partition by org.apache.pig.Identity); " +
                       "C = (sample B 0.49); " +
                       "D = (order C by $0, $1); " +
                       "E = (load 'y' as (d1, d2)); " +
                       "F = (union onschema D, E); " +
                       "G = (load 'z' as (g1:int, g2:tuple(g21, g22))); " +
                       "H = (cross F, G); " +
                       "split H into I if 10 > 5, J if 'world' eq 'hello', K if 77 <= 200; " +
                       "L = (store J into 'output');";
        generateLogicalPlan( query );
    }

    @Test
    public void testCubeBasic() {
	String query = "a = load 'input' as (x:chararray,y:chararray,z:long);"
	        + "b = cube a by cube(x,y);"
	        + "c = foreach b generate flatten(group) as (x,y), COUNT(cube) as count, SUM(cube.z) as total;"
	        + "store c into 'output';";
	generateLogicalPlan(query);
    }

    @Test
    public void testCubeMultipleIAlias() {
	String query = "a = load 'input' as (x:chararray,y:chararray,z:long);"
	        + "a = load 'input' as (x,y:chararray,z:long);"
	        + "a = load 'input' as (x:chararray,y:chararray,z:long);"
	        + "b = cube a by rollup(x,y);"
	        + "c = foreach b generate flatten(group) as (x,y), COUNT(cube) as count, SUM(cube.z) as total;"
	        + "store c into 'c';";
	generateLogicalPlan(query);
    }

    @Test
    public void testCubeAfterForeach() {
	String query = "a = load 'input' as (x:chararray,y:chararray,z:long);"
	        + "b = foreach a generate x as type,y as location,z as number;"
	        + "c = cube b by cube(type,location);"
	        + "d = foreach c generate flatten(group) as (type,location), COUNT(cube) as count, SUM(cube.number) as total;"
	        + "store d into 'd';";
	generateLogicalPlan(query);
    }

    @Test
    public void testFilter() {
        String query = "A = load 'x' as ( u:int, v:long, w:bytearray); " +
                       "B = filter A by 2 > 1;\n" +
                       "store B into 'y';";
        generateLogicalPlan( query );
    }

    @Test
    public void testScopedAlias() {
        String query = "A = load 'x' as ( u:int, v:long, w:bytearray);" +
                       "B = load 'y' as ( u:int, x:int, y:chararray);" +
                       "C = join A by u, B by u;" +
                       "D = foreach C generate A::u, B::u, v, x;" +
                       "store D into 'z';";
        generateLogicalPlan ( query );
    }

    @Test
    public void testConstantWithNegativeValue() {
        String query = "A = load 'x' as ( u:int, v:long, w:bytearray);" +
                       "B = foreach A generate u, { ( -1, -15L, -3.5, -4.03F, -2.3e3 ) };" +
                       "store B into 'y';";
        generateLogicalPlan ( query );
    }

    @Test
    public void testNegative1() {
        String query = "A = load 'x' as ( a : bag{ T:tuple(u, v) }, c : int, d : long );" +
                       "B = foreach A { S = c * 2; T = limit S 100; generate T; };" +
                       "store B into 'y';";
        try {
            ParserTestingUtils.generateLogicalPlan( query );
        } catch(Exception ex) {
            Assert.assertTrue( ex instanceof NonProjectExpressionException );
            return;
        }
        Assert.fail( "Query is supposed to be failing." );
    }

    @Test
    public void testNegative2() {
    	String query = "ship = load 'x';";
    	try {
            ParserTestingUtils.generateLogicalPlan( query );
        } catch(Exception ex) {
            Assert.assertTrue( ex instanceof MismatchedTokenException );
            MismatchedTokenException mex = (MismatchedTokenException)ex;
            Assert.assertTrue( mex.token.getText().equals("ship") );
            return;
        }
        Assert.fail( "Query is supposed to be failing." );
    }

    @Test
    public void testNegative3() {
    	String query = "A = load 'y'; all = load 'x';";
    	try {
            ParserTestingUtils.generateLogicalPlan( query );
        } catch(Exception ex) {
            Assert.assertTrue( ex instanceof MismatchedTokenException );
            MismatchedTokenException mex = (MismatchedTokenException)ex;
            Assert.assertTrue( mex.token.getText().equals("all") );
            return;
        }
        Assert.fail( "Query is supposed to be failing." );
    }

    @Test
    public void testMultilineFunctionArgument() {
        String query = "LOAD 'testIn' \n" +
            "USING PigStorage ('\n');";
        generateLogicalPlan( query );
    }

    @Test
    // See PIG-2320
    public void testInlineOpInGroup() {
        String query = "a = load 'data1' as (x:int); \n" +
            "a_1 = filter (group a by x) by COUNT(a) > 0;";
        generateLogicalPlan( query );
    }

}


text:
Este repositorio tendrá la descripción de los temas que actualmente se enseñan en la materia de IASC, informalmente llamado
Arquitecturas Concurrentes. Esta materia pertenece a la curricula electiva de 5to año de la carrera de Sistemas en la
UTN-FRBA.

Puede verse el temario en el siguiente [índice](/index.md)


text:
package com.rometools.propono.atom.server;

import com.rometools.propono.atom.common.AtomService;
import com.rometools.propono.atom.common.Categories;
import com.rometools.rome.feed.atom.Entry;
import com.rometools.rome.feed.atom.Feed;

/**
 * Interface for handling single Atom protocol requests.
 *
 * <p>
 * To create your own Atom protocol implementation you must implement this interface and create a
 * concrete sub-class of {@link com.rometools.rome.propono.atom.server.AtomHandlerFactory} which is
 * capable of instantiating it.
 * </p>
 */
public interface AtomHandler {
    /**
     * Get username of authenticated user. Return the username of the authenticated user
     */
    public String getAuthenticatedUsername();

    /**
     * Return {@link com.rometools.rome.propono.atom.common.AtomService} object that contains the
     * {@link com.rometools.rome.propono.atom.common.Workspace} objects available to the currently
     * authenticated user and within those the
     * {@link com.rometools.rome.propono.atom.common.Collection} avalaible.
     */
    public AtomService getAtomService(AtomRequest req) throws AtomException;

    /**
     * Get categories, a list of Categories objects
     */
    public Categories getCategories(AtomRequest req) throws AtomException;

    /**
     * Return collection or portion of collection specified by request.
     *
     * @param req Details of HTTP request
     */
    public Feed getCollection(AtomRequest req) throws AtomException;

    /**
     * Store new entry in collection specified by request and return representation of entry as it
     * is stored on server.
     *
     * @param req Details of HTTP request
     * @return Location URL of new entry
     */
    public Entry postEntry(AtomRequest req, Entry entry) throws AtomException;

    /**
     * Get entry specified by request.
     *
     * @param req Details of HTTP request
     */
    public Entry getEntry(AtomRequest req) throws AtomException;

    /**
     * Get media resource specified by request.
     *
     * @param req Details of HTTP request
     */
    public AtomMediaResource getMediaResource(AtomRequest req) throws AtomException;

    /**
     * Update entry specified by request and return new entry as represented on the server.
     *
     * @param req Details of HTTP request
     */
    public void putEntry(AtomRequest req, Entry entry) throws AtomException;

    /**
     * Delete entry specified by request.
     *
     * @param req Details of HTTP request
     */
    public void deleteEntry(AtomRequest req) throws AtomException;

    /**
     * Store media data in collection specified by request, create an Atom media-link entry to store
     * metadata for the new media file and return that entry to the caller.
     *
     * @param req Details of HTTP request
     * @param entry New entry initialzied with only title and content type
     * @return Location URL of new media entry
     */
    public Entry postMedia(AtomRequest req, Entry entry) throws AtomException;

    /**
     * Update the media file part of a media-link entry.
     *
     * @param req Details of HTTP request
     */
    public void putMedia(AtomRequest req) throws AtomException;

    /**
     * Return true if specified request represents URI of a Service Document.
     *
     * @param req Details of HTTP request
     */
    public boolean isAtomServiceURI(AtomRequest req);

    /**
     * Return true if specified request represents URI of a Categories Document.
     *
     * @param req Details of HTTP request
     */
    public boolean isCategoriesURI(AtomRequest req);

    /**
     * Return true if specified request represents URI of a collection.
     *
     * @param req Details of HTTP request
     */
    public boolean isCollectionURI(AtomRequest req);

    /**
     * Return true if specified request represents URI of an Atom entry.
     *
     * @param req Details of HTTP request
     */
    public boolean isEntryURI(AtomRequest req);

    /**
     * Return true if specified patrequesthinfo represents media-edit URI.
     *
     * @param req Details of HTTP request
     */
    public boolean isMediaEditURI(AtomRequest req);
}


text:
var path = require('path');
var webpack = require('webpack');
var DashboardPlugin = require('webpack-dashboard/plugin');
var HtmlWebpackPlugin = require('html-webpack-plugin');
var HtmlWebpackHarddiskPlugin = require('html-webpack-harddisk-plugin');
var PreloadWebpackPlugin = require('preload-webpack-plugin');

module.exports = function(env){

    // default params
    var port = 8080;
    var host = 'localhost';

    // custom configuration options (via cli: 'yarn start -- --env.option')
    if(env){
        if(env.port){
            port = env.port;
        }
        if(env.host){
            host = env.host;
        }
    }

    // return config object
    return {
        entry: {
            main:[
                'react-hot-loader/patch',
                'webpack-dev-server/client?http://' + host + ':' + port,
                'webpack/hot/only-dev-server',
                'babel-polyfill',
                'normalize.css',
                './src/index'
            ]
        },

        output: {
            filename: '[name].js',
            path: path.resolve(__dirname, 'dist/static'),
            publicPath: '/static/'
        },

        devtool: 'cheap-module-eval-source-map',

        module: {
            rules: [
                // linting on each build takes too long...enable it in your IDE instead { enforce: 'pre', test: /\.(js||jsx)$/, exclude: /node_modules/, loader: 'eslint-loader' },
                { test: /\.(js|jsx)$/, use: ['babel-loader'], exclude: /node_modules/ },
                { test: /\.css$/, use: ['style-loader', 'css-loader'] },
                { test: /\.scss$/, use: ['style-loader', 'css-loader?modules&importLoaders=1&localIdentName=[name]__[local]___[hash:base64:5]', 'postcss-loader'] },
                { test: /\.(png|jpg|jpeg|gif|svg|woff|woff2|ttf|eot)$/, use: ['file-loader'] }
            ],
        },

        resolve: {
            modules: [path.resolve(__dirname, './src'), 'node_modules']
        },

        plugins: [
            new webpack.HotModuleReplacementPlugin(),
            new webpack.NamedModulesPlugin(),
            new webpack.NoEmitOnErrorsPlugin(),
            new webpack.optimize.ModuleConcatenationPlugin(),
            new DashboardPlugin(),
            new webpack.DefinePlugin({
                __HMR__: true,
            }),
            new webpack.optimize.CommonsChunkPlugin({
                name: 'vendor',
                minChunks: function (module) {
                    // implicitly split all code from the 'node_modules' directory into this chunk
                    return module.context && module.context.indexOf('node_modules') !== -1;
                }
            }),
            new webpack.optimize.CommonsChunkPlugin({
                name: 'manifest',
                minChunks: Infinity
            }),
            new HtmlWebpackPlugin({
                alwaysWriteToDisk: true,
                title: 'boiler-pack (HMR)',
                favicon: path.resolve(__dirname, 'src/favicon.ico'),
                template: path.resolve(__dirname, 'src/index.template.ejs')
            }),
            new PreloadWebpackPlugin({
                rel: 'prefetch'
            }),
            new HtmlWebpackHarddiskPlugin({
                outputPath: path.resolve(__dirname, 'dist')
            })
        ],

        devServer: {
            historyApiFallback: true,
            host: host,
            port: port,
            hot: true,
            quiet: true,
            contentBase: path.resolve(__dirname, 'dist'),
            publicPath: '/static/'
        },
    }
};