text:
package com.facebook.react.views.modal;

import android.content.DialogInterface;
import android.graphics.Point;
import androidx.annotation.Nullable;
import com.facebook.react.bridge.ReadableArray;
import com.facebook.react.common.MapBuilder;
import com.facebook.react.module.annotations.ReactModule;
import com.facebook.react.uimanager.LayoutShadowNode;
import com.facebook.react.uimanager.ReactStylesDiffMap;
import com.facebook.react.uimanager.StateWrapper;
import com.facebook.react.uimanager.ThemedReactContext;
import com.facebook.react.uimanager.UIManagerHelper;
import com.facebook.react.uimanager.ViewGroupManager;
import com.facebook.react.uimanager.ViewManagerDelegate;
import com.facebook.react.uimanager.annotations.ReactProp;
import com.facebook.react.uimanager.events.EventDispatcher;
import com.facebook.react.viewmanagers.ModalHostViewManagerDelegate;
import com.facebook.react.viewmanagers.ModalHostViewManagerInterface;
import java.util.Map;

/** View manager for {@link ReactModalHostView} components. */
@ReactModule(name = ReactModalHostManager.REACT_CLASS)
public class ReactModalHostManager extends ViewGroupManager<ReactModalHostView>
    implements ModalHostViewManagerInterface<ReactModalHostView> {

  public static final String REACT_CLASS = "RCTModalHostView";

  private final ViewManagerDelegate<ReactModalHostView> mDelegate;

  public ReactModalHostManager() {
    mDelegate = new ModalHostViewManagerDelegate<>(this);
  }

  @Override
  public String getName() {
    return REACT_CLASS;
  }

  @Override
  protected ReactModalHostView createViewInstance(ThemedReactContext reactContext) {
    return new ReactModalHostView(reactContext);
  }

  @Override
  public LayoutShadowNode createShadowNodeInstance() {
    return new ModalHostShadowNode();
  }

  @Override
  public Class<? extends LayoutShadowNode> getShadowNodeClass() {
    return ModalHostShadowNode.class;
  }

  @Override
  public void onDropViewInstance(ReactModalHostView view) {
    super.onDropViewInstance(view);
    view.onDropInstance();
  }

  @Override
  @ReactProp(name = "animationType")
  public void setAnimationType(ReactModalHostView view, @Nullable String animationType) {
    if (animationType != null) {
      view.setAnimationType(animationType);
    }
  }

  @Override
  @ReactProp(name = "transparent")
  public void setTransparent(ReactModalHostView view, boolean transparent) {
    view.setTransparent(transparent);
  }

  @Override
  @ReactProp(name = "statusBarTranslucent")
  public void setStatusBarTranslucent(ReactModalHostView view, boolean statusBarTranslucent) {
    view.setStatusBarTranslucent(statusBarTranslucent);
  }

  @Override
  @ReactProp(name = "hardwareAccelerated")
  public void setHardwareAccelerated(ReactModalHostView view, boolean hardwareAccelerated) {
    view.setHardwareAccelerated(hardwareAccelerated);
  }

  @Override
  @ReactProp(name = "visible")
  public void setVisible(ReactModalHostView view, boolean visible) {
    // iOS only
  }

  @Override
  public void setPresentationStyle(ReactModalHostView view, @Nullable String value) {}

  @Override
  public void setAnimated(ReactModalHostView view, boolean value) {}

  @Override
  public void setSupportedOrientations(ReactModalHostView view, @Nullable ReadableArray value) {}

  @Override
  public void setIdentifier(ReactModalHostView view, int value) {}

  @Override
  protected void addEventEmitters(
      final ThemedReactContext reactContext, final ReactModalHostView view) {
    final EventDispatcher dispatcher =
        UIManagerHelper.getEventDispatcherForReactTag(reactContext, view.getId());
    if (dispatcher != null) {
      view.setOnRequestCloseListener(
          new ReactModalHostView.OnRequestCloseListener() {
            @Override
            public void onRequestClose(DialogInterface dialog) {
              dispatcher.dispatchEvent(
                  new RequestCloseEvent(UIManagerHelper.getSurfaceId(reactContext), view.getId()));
            }
          });
      view.setOnShowListener(
          new DialogInterface.OnShowListener() {
            @Override
            public void onShow(DialogInterface dialog) {
              dispatcher.dispatchEvent(
                  new ShowEvent(UIManagerHelper.getSurfaceId(reactContext), view.getId()));
            }
          });
      view.setEventDispatcher(dispatcher);
    }
  }

  @Override
  public Map<String, Object> getExportedCustomDirectEventTypeConstants() {
    return MapBuilder.<String, Object>builder()
        .put(RequestCloseEvent.EVENT_NAME, MapBuilder.of("registrationName", "onRequestClose"))
        .put(ShowEvent.EVENT_NAME, MapBuilder.of("registrationName", "onShow"))
        .build();
  }

  @Override
  protected void onAfterUpdateTransaction(ReactModalHostView view) {
    super.onAfterUpdateTransaction(view);
    view.showOrUpdate();
  }

  @Override
  public Object updateState(
      ReactModalHostView view, ReactStylesDiffMap props, @Nullable StateWrapper stateWrapper) {
    view.getFabricViewStateManager().setStateWrapper(stateWrapper);
    Point modalSize = ModalHostHelper.getModalHostSize(view.getContext());
    view.updateState(modalSize.x, modalSize.y);
    return null;
  }

  @Override
  public ViewManagerDelegate<ReactModalHostView> getDelegate() {
    return mDelegate;
  }
}


text:
/*global angular, jQuery*/
(function ($) {
    angular
        .module('simplAdmin.pricing')
        .controller('CartRuleFormCtrl', ['$state', '$stateParams', 'cartRuleService', 'translateService', CartRuleFormCtrl]);

    function CartRuleFormCtrl($state, $stateParams, cartRuleService, translateService) {
        var vm = this;
        vm.translate = translateService;
        vm.cartRule = { isCouponRequired: true };
        vm.cartRule.products = [];
        vm.cartRuleId = $stateParams.id;
        vm.isEditMode = vm.cartRuleId > 0;

        vm.datePickerStartOn = {};
        vm.datePickerEndOn = {};

        vm.openCalendar = function (e, picker) {
            vm[picker].open = true;
        };

        vm.save = function save() {
            var promise;
            if (vm.isEditMode) {
                promise = cartRuleService.editCartRule(vm.cartRule);
            } else {
                promise = cartRuleService.createCartRule(vm.cartRule);
            }

            promise
                .then(function (result) {
                    $state.go('cart-rules');
                })
                .catch(function (response) {
                    var error = response.data;
                    vm.validationErrors = [];
                    if (error && angular.isObject(error)) {
                        for (var key in error) {
                            vm.validationErrors.push(error[key][0]);
                        }
                    } else {
                        vm.validationErrors.push('Could not add cartRule.');
                    }
                });
        };

        function init() {
            if (vm.isEditMode) {
                cartRuleService.getCartRule(vm.cartRuleId).then(function (result) {
                    vm.cartRule = result.data;
                });
            }
        }

        init();
    }
})(jQuery);

redpajama_set_name: RedPajamaGithub


text:
{-# LANGUAGE TemplateHaskell #-}
module Language.Java.Lenses.Syntax.Types where

import           Language.Java.Lenses.CustomLenses
import           Language.Java.Syntax.Types

allLenses ''Type

allLenses ''RefType

allLenses ''ClassName

allLenses ''Package

allLenses ''TypeArgument

allLenses ''TypeDeclSpecifier

allLenses ''WildcardBound

allLenses ''PrimType

allLenses ''TypeParam

allLenses ''Ident

allLenses ''Name

allLenses ''RelaxedType


text:
package com.parikls.movieland.web.controller.dto.auth;

public class LogoutCredentials {

    private String token;

    public String getToken() {
        return token;
    }

    public void setToken(String token) {
        this.token = token;
    }
}


text:
using OpenQA.Selenium;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;

namespace DesignPattern.Pages.SelectablePage
{
    public partial class SelectablePage
    {
        public IWebElement firstItem
        {
            get
            {
                return Driver.FindElement(By.XPath("//*[@id=\"selectable\"]/li[1]"));
            }
        }

        public IWebElement secondItem
        {
            get
            {
                return Driver.FindElement(By.XPath("//*[@id=\"selectable\"]/li[2]"));
            }
        }
    }
}


text:
/**
 *
 * Apply middle click on an element. If selector is not provided, click on the last
 * moved-to location.
 *
 * @param {String} selector element to click on
 * @uses protocol/element, protocol/buttonPress
 * @type action
 *
 */

var handleMouseButtonCommand = require('../helpers/handleMouseButtonCommand');

module.exports = function middleClick (cssSelector) {
    return handleMouseButtonCommand.call(this, cssSelector, 'middle');
};


text:
@import CoreLocation;

@interface CLLocation (WMFComparison)

- (BOOL)wmf_isEqual:(CLLocation *)location;

@end


text:
/***************************************************************************

Copyright (c) Microsoft Corporation. All rights reserved.
This code is licensed under the Visual Studio SDK license terms.
THIS CODE IS PROVIDED *AS IS* WITHOUT WARRANTY OF
ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING ANY
IMPLIED WARRANTIES OF FITNESS FOR A PARTICULAR
PURPOSE, MERCHANTABILITY, OR NON-INFRINGEMENT.

***************************************************************************/

using System;
using System.Collections.Generic;
using System.Text;

using System.CodeDom.Compiler;
using IronPython.CodeDom;

namespace Microsoft.Samples.VisualStudio.IronPythonTasks
{
	/// <summary>
	/// This class is an alternate compiler to build Iron Python project.
	/// The reason we have it is that the standard compiler produces assemblies
	/// which are meant to be interpreted rather then ran directly, and WAP scenarios
	/// require real assemblies.
	/// </summary>
	internal class ExperimentalCompiler : ICompiler
	{
		#region fields
		private List<string> sourceFiles;
		private string outputAssembly;
		private IronPython.Hosting.CompilerSink errorSink;
		private List<string> referencedAssemblies = new List<string>();
		private IList<IronPython.Hosting.ResourceFile> resourceFiles = new List<IronPython.Hosting.ResourceFile>();
		private string mainFile = null;
		private System.Reflection.Emit.PEFileKinds targetKind = System.Reflection.Emit.PEFileKinds.Dll;
		private bool includeDebugInformation = false;
		#endregion

		#region Constructors
		public ExperimentalCompiler(IList<string> sourcesFiles, string outputAssembly)
		{
			this.sourceFiles = (List<string>)sourcesFiles;
			this.outputAssembly = outputAssembly;
		}

		public ExperimentalCompiler(IList<string> sourcesFiles, string outputAssembly, IronPython.Hosting.CompilerSink compilerSink)
		{
			this.sourceFiles = (List<string>)sourcesFiles;
			this.outputAssembly = outputAssembly;
			this.errorSink = compilerSink;
		}
		#endregion

		#region ICompiler Members

		public IList<string> SourceFiles
		{
			get
			{
				return this.sourceFiles;
			}
			set
			{
				this.sourceFiles = (List<string>)value;
			}
		}

		public string OutputAssembly
		{
			get
			{
				return this.outputAssembly;
			}
			set
			{
				this.outputAssembly = value;
			}
		}

		public IList<string> ReferencedAssemblies
		{
			get
			{
				return this.referencedAssemblies;
			}
			set
			{
				this.referencedAssemblies = (List<string>)value;
			}
		}

		public IList<IronPython.Hosting.ResourceFile> ResourceFiles
		{
			get
			{
				return this.resourceFiles;
			}
			set
			{
				this.resourceFiles = value;
			}
		}

		public string MainFile
		{
			get
			{
				return mainFile;
			}
			set
			{
				this.mainFile = value;
			}
		}

		public System.Reflection.Emit.PEFileKinds TargetKind
		{
			get
			{
				return this.targetKind;
			}
			set
			{
				this.targetKind = value;
			}
		}

		public bool IncludeDebugInformation
		{
			get
			{
				return this.includeDebugInformation;
			}
			set
			{
				this.includeDebugInformation = value;
			}
		}

		public void Compile()
		{
			PythonProvider provider = new PythonProvider();
			CompilerParameters options = new CompilerParameters(referencedAssemblies.ToArray(), OutputAssembly, IncludeDebugInformation);
			options.MainClass = MainFile;
			foreach(IronPython.Hosting.ResourceFile resourceInfo in resourceFiles)
			{
				// NOTE: with this approach we lack a way to control the name of the generated resource or if it is public
				string resource = resourceInfo.File;
				options.EmbeddedResources.Add(resource);
			}

			CompilerResults results = provider.CompileAssemblyFromFile(options, sourceFiles.ToArray());
			foreach (CompilerError error in results.Errors)
			{
				int errorNumber = 0;
				int.TryParse(error.ErrorNumber, out errorNumber);
				this.errorSink.AddError(error.FileName, error.ErrorText, String.Empty, new IronPython.Hosting.CodeSpan(error.Line, error.Column, error.Line, error.Column+1), errorNumber, error.IsWarning ? IronPython.Hosting.Severity.Warning : IronPython.Hosting.Severity.Error);
			}
		}

		#endregion
	}
}


text:
<!DOCTYPE html>

    <meta charset="utf-8">
    <meta http-equiv="X-UA-Compatible" content="IE=edge,chrome=1">
    <title>Interface ILoadsModel
   </title>
    <meta name="viewport" content="width=device-width">
    <meta name="title" content="Interface ILoadsModel
   ">
    <meta name="generator" content="docfx 2.56.6.0">

    <link rel="shortcut icon" href="../favicon.ico">
    <link rel="stylesheet" href="../styles/docfx.vendor.css">
    <link rel="stylesheet" href="../styles/docfx.css">
    <link rel="stylesheet" href="../styles/main.css">
    <meta property="docfx:navrel" content="../toc.html">
    <meta property="docfx:tocrel" content="toc.html">

              <button type="button" class="navbar-toggle" data-toggle="collapse" data-target="#navbar">
                Toggle navigation

              </button>

              <form class="navbar-form navbar-right" role="search" id="search">

                  <input type="text" class="form-control" id="search-query" placeholder="Search" autocomplete="off">

              </form>

          Show / Hide Table of Contents

Interface ILoadsModel

An object which can load a model object from a file path.

Namespace: ZptSharp.Cli

Assembly: ZptSharp.Cli.dll

Syntax

public interface ILoadsModel

Methods

    |
    Improve this Doc

    View Source

LoadModelAsync(String, CancellationToken)

Loads and returns the model.

Declaration

Task<object> LoadModelAsync(string path, CancellationToken token = default(CancellationToken))

Parameters

        Type
        Name
        Description

        System.String
        path

The file path.

        System.Threading.CancellationToken
        token

An optional cancallation token.

Returns

        Type
        Description

        System.Threading.Tasks.Task<System.Object>

The model object.

                    Improve this Doc

                    View Source

In This Article

              Back to top

            Generated by DocFX


text:
package org.callimachusproject.interceptors;

import java.io.IOException;
import java.lang.reflect.Method;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.LinkedHashMap;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import java.util.regex.PatternSyntaxException;

import javax.activation.MimeType;
import javax.activation.MimeTypeParseException;

import org.apache.http.HttpException;
import org.apache.http.HttpRequest;
import org.apache.http.HttpResponse;
import org.apache.http.RequestLine;
import org.apache.http.message.BasicHttpRequest;
import org.apache.http.protocol.HttpContext;
import org.callimachusproject.annotations.rel;
import org.callimachusproject.annotations.title;
import org.openrdf.annotations.Path;
import org.openrdf.annotations.Type;
import org.openrdf.http.object.chain.HttpRequestChainInterceptor;
import org.openrdf.http.object.helpers.ObjectContext;
import org.openrdf.http.object.helpers.ResourceTarget;

/**
 * Add an HTTP Header called 'Link' with other operations available to this
 * resource.
 *
 * @author James Leigh
 *
 */
public class LinksFilter implements HttpRequestChainInterceptor {
	private static final Pattern STRIP_REGEX = Pattern
			.compile("(?<!^|\\\\)[\\.\\$\\|\\(\\)\\[\\]\\{\\}\\^\\?\\*\\+]|\\\\(.)");
	private final Pattern SIMPLE = Pattern
			.compile("^\\??([^\\.\\$\\|\\[\\]\\{\\}\\^\\?\\*\\+\\\\]|\\\\[^a-zA-Z0-9])*");

	@Override
	public HttpResponse intercept(HttpRequest request, HttpContext context)
			throws HttpException, IOException {
		return null;
	}

	@Override
	public void process(HttpRequest request, HttpResponse response,
			HttpContext context) throws HttpException, IOException {
		ObjectContext ctx = ObjectContext.adapt(context);
		String uri = request.getRequestLine().getUri();
		ResourceTarget resource = ctx.getResourceTarget();
		String iri = resource.getTargetObject().getResource().stringValue();
		if (iri.endsWith(uri)) {
			// no extra path or query string
			for (String link : getRelLinks(request, resource)) {
				response.addHeader("Link", link);
			}
		} else {
			Method handler = getHandlerMethod(request, resource);
			if (handler != null) {
				for (String link : getRevLinks(resource, handler)) {
					response.addHeader("Link", link);
				}
			}
		}
	}

	private List<String> getRelLinks(HttpRequest request, ResourceTarget resource) {
		Map<String, List<Method>> map = new LinkedHashMap<String, List<Method>>();
		for (java.lang.reflect.Method method : resource.getTargetObject().getClass().getMethods()) {
			org.openrdf.annotations.Method m = method.getAnnotation(org.openrdf.annotations.Method.class);
			if (m == null || !Arrays.asList(m.value()).contains("GET"))
				continue;
			Path p = method.getAnnotation(Path.class);
			if (p == null) {
				putAdd(map, "", method);
			} else {
				for (String regex : p.value()) {
					String strip = strip(regex);
					if (strip == null || strip.length() < 1)
						continue;
					Matcher match = SIMPLE.matcher(regex);
					try {
						if (match.matches() || match.find()
								&& Pattern.matches(regex, strip)) {
							putAdd(map, strip, method);
						}
					} catch (PatternSyntaxException e) {
						// ignore
					}
				}
			}
		}
		String iri = resource.getTargetObject().getResource().stringValue();
		List<String> result = new ArrayList<String>();
		for (Map.Entry<String, List<Method>> e : map.entrySet()) {
			String suffix = e.getKey();
			List<Method> methods = e.getValue();
			getLinks(iri, suffix, false, methods, result);
		}
		return result;
	}

	private String strip(String regex) {
		return STRIP_REGEX.matcher(regex).replaceAll("$1");
	}

	private <T> void putAdd(Map<String, List<T>> map, String suffix, T method) {
		List<T> list = map.get(suffix);
		if (list == null) {
			map.put(suffix, list = new ArrayList<T>());
		}
		list.add(method);
	}

	private List<String> getRevLinks(ResourceTarget resource, Method handler) {
		Path p = handler.getAnnotation(Path.class);
		if (p == null)
			return Collections.emptyList();
		List<String> paths = Arrays.asList(p.value());
		List<Method> list = new ArrayList<Method>();
		for (java.lang.reflect.Method method : resource.getTargetObject().getClass().getMethods()) {
			org.openrdf.annotations.Method m = method.getAnnotation(org.openrdf.annotations.Method.class);
			if (m == null || !Arrays.asList(m.value()).contains("GET"))
				continue;
			Path path = method.getAnnotation(Path.class);
			if (path != null && Arrays.asList(path.value()).containsAll(paths)) {
				list.add(method);
			}
		}
		if (list.isEmpty())
			return Collections.emptyList();
		String iri = resource.getTargetObject().getResource().stringValue();
		List<String> result = new ArrayList<String>(list.size());
		getLinks(iri, null, true, list, result);
		return result;
	}

	private Method getHandlerMethod(HttpRequest request, ResourceTarget target) {
		Method method = target.getHandlerMethod(request);
		RequestLine line = request.getRequestLine();
		if (method != null || !"HEAD".equals(line.getMethod()))
			return method;
		HttpRequest get = new BasicHttpRequest("GET", line.getUri(), line.getProtocolVersion());
		get.setHeaders(request.getAllHeaders());
		return target.getHandlerMethod(get);
	}

	private void getLinks(String uri, String suffix, boolean rev,
			List<Method> methods, List<String> result) {
		Set<String> rels = new LinkedHashSet<String>();
		Set<String> types = new LinkedHashSet<String>();
		Set<String> titles = new LinkedHashSet<String>();
		for (Method m : methods) {
			Collection<String> mrel = getMethodRel(m);
			if (!mrel.isEmpty()) {
				rels.addAll(mrel);
				for (String type : getResponseType(m)) {
					try {
						MimeType mtype = new MimeType(type);
						mtype.removeParameter("q");
						types.add(mtype.toString());
					} catch (MimeTypeParseException e) {
						types.add(type);
					}
				}
				titles.addAll(getMethodTitles(m));
			}
		}
		if (!rels.isEmpty()) {
			result.add(serialize(uri, suffix, rev, rels, types, titles));
		}
	}

	private String serialize(String uri, String suffix, boolean rev,
			Set<String> rels, Set<String> types, Set<String> titles) {
		StringBuilder sb = new StringBuilder();
		sb.append("<").append(uri);
		if (suffix != null && suffix.length() > 0) {
			sb.append(suffix);
		}
		sb.append(">");
		if (rev) {
			sb.append("; rev=\"");
		} else {
			sb.append("; rel=\"");
		}
		for (String rel : rels) {
			sb.append(rel).append(" ");
		}
		sb.setCharAt(sb.length() - 1, '"');
		if (!types.isEmpty()) {
			sb.append("; type=\"");
			for (String type : types) {
				sb.append(type).append(" ");
			}
			sb.setCharAt(sb.length() - 1, '"');
		}
		for (String title : titles) {
			sb.append("; title=\"").append(title).append("\"");
		}
		return sb.toString();
	}

	private Collection<String> getMethodRel(Method method) {
		if (method.isAnnotationPresent(rel.class))
			return Arrays.asList(method.getAnnotation(rel.class).value());
		return Collections.emptyList();
	}

	private Collection<String> getMethodTitles(Method method) {
		if (method.isAnnotationPresent(title.class))
			return Arrays.asList(method.getAnnotation(title.class).value());
		return Collections.emptyList();
	}

	private Collection<String> getResponseType(Method m) {
		Collection<String> set = new LinkedHashSet<String>();
		Type ann = m.getAnnotation(Type.class);
		if (ann != null) {
			set.addAll(Arrays.asList(ann.value()));
		}
		return set;
	}

}


text:
<?php

namespace Kirby\StaticBuilder;

use C;
use R;
use Response;

class Controller
{
    /** @var bool */
    private static $registered = false;

    /**
     * Register StaticBuilder's routes
     * Returns a string with the current status
     * @return string
     */
    static function register()
    {
        if (static::$registered) {
            return 'StaticBuilder is already enabled';
        }
        if (!C::get('staticbuilder', false)) {
            return 'StaticBuilder does not seem to be enabled in your config. See https://github.com/fvsch/kirby-staticbuilder/blob/master/doc/install.md for instructions.';
        }
        $kirby = kirby();
        if (!class_exists('Kirby\\Registry')) {
            throw new Exception('Twig plugin requires Kirby 2.3 or higher. Current version: ' . $kirby->version());
        }
        $kirby->set('route', [
            'pattern' => 'staticbuilder',
            'action' => 'Kirby\\StaticBuilder\\Controller::siteAction',
            'method' => 'GET|POST'
        ]);
        $kirby->set('route', [
            'pattern' => 'staticbuilder/(:all)',
            'action' => 'Kirby\\StaticBuilder\\Controller::pageAction',
            'method' => 'GET|POST'
        ]);
        static::$registered = true;
        return 'Enabled StaticBuilder routes.';
    }

    /**
     * Kirby router action that lists all pages to build and files to copy,
     * and performs the actual build on user confirmation.
     * @return Response
     */
    static function siteAction()
    {
        $write = R::is('POST') and R::get('confirm');
        $site = site();
        $kirby = kirby();
        $builder = new Builder();

        // bail if cache is active
        if ($kirby->get('option', 'cache')) {
            return $builder->htmlReport([
                'mode' => 'fatal',
                'error' => 'Configuration error',
                'errorTitle' => 'Disable Kirby's cache',
                'errorDetails' => '
StaticBuilder requires Kirby's cache option to be disabled.
' .
                    '
<?php // site/config/config.localhost.php' . "\n" .
                    'c::set(\'cache\', false);
',

                'confim' => false,
                'summary' => []
            ]);
        }

        // try a build
        $builder->run($site, $write);

        // this may not run if we've had errors
        return $builder->htmlReport([
            'mode'    => 'site',
            'error'   => false,
            'confirm' => $write,
            'summary' => $builder->summary
        ]);
    }

    /**
     * Similar to siteAction but for a single page.
     * @param $uri
     * @return Response
     */
    static function pageAction($uri)
    {
        $write = R::is('POST') and R::get('confirm');
        $page = page($uri);
        $builder = new Builder();
        $data = [
            'mode'    => 'page',
            'error'   => false,
            'confirm' => $write,
            'summary' => []
        ];
        if (!$page) {
            $data['error'] = "Error: Cannot find page for \"$uri\"";
        }
        else {
            $builder->run($page, $write);
            $data['summary'] = $builder->summary;
        }
        return $builder->htmlReport($data);
    }
}


text:
package z_archive.jena_1746_tdb2_cache;

import static org.apache.jena.tdb2.sys.StoreConnection.connectCreate;
import static org.junit.Assert.assertEquals;

import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.ArrayList;
import java.util.List;
import java.util.UUID;

import org.apache.jena.dboe.base.file.Location;
import org.apache.jena.query.Dataset;
import org.apache.jena.query.DatasetFactory;
import org.apache.jena.query.TxnType;
import org.apache.jena.rdf.model.*;
import org.apache.jena.tdb2.params.StoreParams;
import org.apache.jena.tdb2.params.StoreParamsBuilder;
import org.apache.jena.vocabulary.RDFS;
import org.junit.Test;

public class TDB2_Rollbacktest_Original {

      public static final String GRAPH_NAME = "http://virzrt.hu/test-model";
      public static final String FILL_GRAPH_NAME = "http://virzrt.hu/test-model-fill";
      public static final String RES_NAME = "http://virzrt.hu/test-res#";
      public static final String PROP_NAME = "http://virzrt.hu/test-prop#";

      @Test
      public void testRollbackCacheFail() throws IOException {
        final Path tempDirWithPrefix = Files.createTempDirectory("jenaDatabaseRollbackTest");

        // Make the cache small enough

        final StoreParams params =
            StoreParamsBuilder.create(StoreParams.getDftStoreParams())
                .node2NodeIdCacheSize(26)
                .nodeId2NodeCacheSize(26)
                .build();
        final Dataset tdb =
            DatasetFactory.wrap(
                connectCreate(Location.create(tempDirWithPrefix.toAbsolutePath().toString()), params)
                    .getDatasetGraph());

        // Create some fix resources and properties:
        Resource[] resources = new Resource[3];
        for (int i = 0; i < 3; i++) {
          resources[i] = ResourceFactory.createResource(RES_NAME + Integer.toString(i));
        }

        Property[] properties = new Property[2];
        for (int i = 0; i < 2; i++) {
          properties[i] = ResourceFactory.createProperty(PROP_NAME + Integer.toString(i));
        }

        // Create a graph to store and flush cache if necessary

        tdb.begin(TxnType.WRITE);
        Model fillModel = tdb.getNamedModel(FILL_GRAPH_NAME);
        for (int i = 0; i < 40; i++) {
          fillModel.add(fillModel.createResource(), RDFS.comment, Integer.toString(i));
        }
        tdb.commit();

        // ********************************************/
        // Transaction to fill cache, but rollbacked:

        tdb.begin(TxnType.WRITE);

        Model namedModel = tdb.getNamedModel(GRAPH_NAME);
        for (int i = 0; i < resources.length; i++) {
          for (int j = 0; j < properties.length; j++) {

            // This method creates some nodes: resource, property and an anonymous node and the value.
            // It fills Graph_NAME (1), resources (4) and properties (2), anonym (8) and value nodes (8)
            addInfoToModel(namedModel, resources[i], properties[j]);
          }
        }

        assertEquals(namedModel.size(), 2 * resources.length * properties.length);
        // Cache is nearly filled, but we discard the real data
        tdb.abort();

        // *************************
        // Abnormal: the cache in NodeTableCache is filled up with invalid node id-s

        tdb.begin(TxnType.WRITE);

        // Create some more date to make some offset in the node data file. It creates 8 data in cache,
        // but
        // reuses some id.

        fillModel = tdb.getNamedModel(FILL_GRAPH_NAME);
        for (int i = 0; i < 8; i++) {
          fillModel.add(fillModel.createResource(), RDFS.comment, Integer.toString(i));
        }

        namedModel = tdb.getNamedModel(GRAPH_NAME);
        // Rebuild the graph, but the order is backward, and anonym items are new.
        for (int i = resources.length - 1; i >= 0; i--) {
          for (int j = properties.length - 1; j >= 0; j--) {
            addInfoToModel(namedModel, resources[i], properties[j]);
          }
        }
        final long size = namedModel.size();
        assertEquals(size, resources.length * properties.length * 2);

        // There is a big chance to flush out some entry in the cache, but can not read it back
        // as these entries was buggy, and pointed to wrong id.

        fillModel = tdb.getNamedModel(FILL_GRAPH_NAME);
        List<Statement> flushList = new ArrayList<>();
        fillModel.listStatements().forEachRemaining(flushList::add);
        assertEquals(flushList.size(), 48);

        // Cached entries flushed out, check again the size:
        // Nothing changed in namedModel, so size should be unchanged
        assertEquals(namedModel.size(), size); //  *** FAILS! sizes() gives 0!

        // ****** This FAILS too
        List<Statement> readback = new ArrayList<>();
        namedModel.listStatements().forEachRemaining(readback::add);
        assertEquals(readback.size(), size);

        // **** Writing out the model UNEXPECTEDLY causes Thrift error, as it reads the whole model.
        tdb.getUnionModel().write(System.out, "TURTLE");

        tdb.commit();

        tdb.close();
      }

      private void addInfoToModel(
          final Model namedModel, final Resource resource, final Property property) {
        final Resource anon = namedModel.createResource();
        namedModel.add(resource, property, anon);
        namedModel.add(anon, property, UUID.randomUUID().toString());
      }
    }


text:
<?php

require_once(DOCROOT . '/runtime/StaticSchema.php');
require_once(DOCROOT . '/runtime/StaticConstraint.php');
require_once(DOCROOT . '/runtime/StaticLink.php');

define('__DOCUMENT_SCHEMA_PATH__', '/org/wrml/core/model/' . Context::DOCUMENT_SCHEMA_NAME);
define('__JSON_FORMAT_ID__', Context::FORMAT_API_DOCROOT . '/application/json');

class Context {

    const PHP_FILE_EXTENSION = '.php';
    const DOCUMENT_SCHEMA_NAME = 'Document';

    const DOCUMENT_SCHEMA_PATH = __DOCUMENT_SCHEMA_PATH__;

    const FORMAT_API_DOCROOT = 'http://api.formats.wrml.org';
    const CONSTRAINT_API_DOCROOT = 'http://api.constraints.wrml.org';
    const RELATIONS_API_DOCROOT = 'http://api.relations.wrml.org';
    const VALIDATORS_API_DOCROOT = 'http://api.validators.wrml.org';

    const JSON_FORMAT_ID = __JSON_FORMAT_ID__;

    public function getRequestPath() {
        if(isset($_SERVER['REQUEST_URI'])){
            $path = $_SERVER['REQUEST_URI'];
        }
        else{
            $path = $_SERVER['PHP_SELF'];
        }

        return $path;
    }

    public function getRequestUri($path) {
        $s = empty($_SERVER["HTTPS"]) ? '' : ($_SERVER["HTTPS"] == "on") ? "s" : "";
        $protocol = $this->strleft(strtolower($_SERVER["SERVER_PROTOCOL"]), "/") . $s;
        $port = ($_SERVER["SERVER_PORT"] == "80") ? "" : (":" . $_SERVER["SERVER_PORT"]);
        return $protocol . "://" . $_SERVER['SERVER_NAME'] . $port . $path;
    }

    public function setHeaders($responseEntity) {

        // TODO: Check the Accept header before setting the Content-Type

        if ($responseEntity instanceof StaticSchema) {
            $schema = $responseEntity;

            // TODO: Implement support for Formats
            //header("Content-type: application/wrml; schema=\"" . $schema->getId() . "\"; format=\"" . //Context::JSON_FORMAT_ID . "\"");

            header("Content-type: application/wrml; schema=\"" . $this->getRequestUri('/org/wrml/core/model/schema/Schema') . "\"");

        }
        else {
            header('Content-type: application/json');
        }

        // TODO: Remove
        //header('Content-type: application/json');

    }

    /**
     * Example usage:
     *
     *    $mime = getBestSupportedMimeType(Array ('application/xhtml+xml', 'text/html'));
     */
    function getBestSupportedMimeType($mimeTypes = null) {
        // Values will be stored in this array
        $AcceptTypes = Array ();

        // Accept header is case insensitive, and whitespace isn't important
        $accept = strtolower(str_replace(' ', '', $_SERVER['HTTP_ACCEPT']));
        // divide it into parts in the place of a ","
        $accept = explode(',', $accept);
        foreach ($accept as $a) {
            // the default quality is 1.
            $q = 1;
            // check if there is a different quality
            if (strpos($a, ';q=')) {
                // divide "mime/type;q=X" into two parts: "mime/type" i "X"
                list($a, $q) = explode(';q=', $a);
            }
            // mime-type $a is accepted with the quality $q
            // WARNING: $q == 0 means, that mime-type isn't supported!
            $AcceptTypes[$a] = $q;
        }
        arsort($AcceptTypes);

        // if no parameter was passed, just return parsed data
        if (!$mimeTypes) return $AcceptTypes;

        $mimeTypes = array_map('strtolower', (array)$mimeTypes);

        // let's check our supported types:
        foreach ($AcceptTypes as $mime => $q) {
           if ($q && in_array($mime, $mimeTypes)) return $mime;
        }
        // no mime-type found
        return null;
    }

    public function createResponseEntity() {

        $requestBody = file_get_contents('php://input');
        $requestPath = $this->getRequestPath();
        $requestUri = $this->getRequestUri($requestPath);
        $requestFilePath = DOCROOT . $requestPath;

        if (is_dir($requestFilePath)) {
            // TODO: Handle collections
            $responseEntity = "TODO: " . $requestUri . " is a Collection.";
        }
        else {

            $requestScriptPath = $requestFilePath . Context::PHP_FILE_EXTENSION;

            if (file_exists($requestScriptPath)) {
                include_once($requestScriptPath);
                $className = basename($requestPath);
                $responseEntity = new $className($this, $requestUri);
            }
            else {
                $schema = new StaticSchema($this, $requestUri);
                $baseSchemaIds = &$schema->getBaseSchemaIds();
                $baseSchemaIds[] = $this->getRequestUri(Context::DOCUMENT_SCHEMA_PATH);
                $schema->setDescription('An autogenerated Schema.');
                $responseEntity = $schema;
            }
        }

        if ($responseEntity instanceof StaticSchema) {
            $responseEntity->setId($requestUri);
        }

        return $responseEntity;
    }

    public function createLink($relPath, $href) {
        $relId = Context::RELATIONS_API_DOCROOT . $relPath;
        $link = new StaticLink($relId, $href);
        return $link;
    }

    public function createConstraint($constraintPath, $description) {
        $constraintId = Context::CONSTRAINT_API_DOCROOT . $constraintPath;
        $constraint = new StaticConstraint($constraintId);
        $constraint->setDescription($description);
        return $constraint;
    }

    public function createUriTextSyntaxConstraint() {
        return $this->createConstraint('/' . Type::TEXT_TYPE . '/Syntax?validator=' . urlencode(Context::VALIDATORS_API_DOCROOT . '/org/wrml/util/validators/UriSyntaxValidator'), 'Constrains the text to conform to the syntax of a URI (see http://www.ietf.org/rfc/rfc3986.txt)');
    }

    public function createSchemaConstraint($schemId) {
        return $this->createConstraint('/' . Type::MODEL_TYPE . '?schema=\"' . urlencode($schemId) . '\"', 'Constrains the model to conform to a specified schema.');
    }

    public function createChoiceMenuConstraint($choiceMenuId) {
        return $this->createConstraint('/' . Type::CHOICE_TYPE . '?menu=\"' . urlencode($choiceMenuId) . '\"', 'Constrains the choice to one of the menu\'s selections.');
    }

    function strleft($s1, $s2) {
        return substr($s1, 0, strpos($s1, $s2));
    }

}


text:
layout: post
title: "\"We should look closely at what Adam Smith actually believed\""
date: 2018-01-21 21:22:08 -0500
external-url: https://aeon.co/essays/we-should-look-closely-at-what-adam-smith-actually-believed
mirror:
---

What we all assume about Adam Smith's seminal contributions to economics
is wrong. _The Wealth of Nations_ was a "very violent attack ... upon the
whole commercial system of Great Britain." That system was the mercantile
system. A handful of merchants controlled all business endeavors in Great
Britain in the late 18th century when Smith was writing. There was great
concentration of power in each sector, much like there is today. Sagar
writes in this piece for _Aeon_, "According to Smith, the most pressing
dangers came not from the state acting alone, but the state when captured
by merchant elites."

That passage about the "invisible hand" that we hear touted by
neoliberalists? Sagar writes, "in the passage of *The Wealth of Nations*
where he invoked the idea of the invisible hand, the immediate context was
not simply that of state intervention in general, but of state intervention
*undertaken at the behest of merchant elites* who were furthering their own
interests at the expense of the public." He continues:

> It is an irony of history that Smith's most famous idea is now usually
> invoked as a defence of unregulated markets in the face of state
> interference, so as to protect the interests of private capitalists. For
> this is roughly the opposite of Smith's original intention, which was to
> advocate for restrictions on what groups of merchants could do. When he
> argued that markets worked remarkably efficiently – because, although
> each individual 'intends only his own gain, and he is in this, as in many
> other cases, led by an invisible hand to promote an end which was no part
> of his intention' – this was an appeal to *free* individuals from the
> constraints imposed upon them by the monopolies that the merchants had
> established, and were using state power to uphold. The invisible hand was
> originally invoked not to draw attention to the problem of state
> *intervention*, but of state *capture*.

How did Smith feel about regulation? He thought it was absolutely
necessary:

>  The true 'science of a statesman or legislator' consisted in deciding
>  how best to govern the merchants' nefarious activities. Effective
>  politicians had to strike a balance between granting economic elites the
>  liberty to pursue legitimate commercial activities, while also applying
>  control when such activities became vehicles for exploitation.

Sagar shrewdly warns the reader to beg off any idea that Smith was
sympathetic to socialist or communist ideas. He was not. But, he was not a
Reagan/Thatcher neoliberal either. No preconceived plan -- no theories for
structuring economies, whether well-regulated or completely unregulated --
could work. Sagar clarifies:

> What Smith is saying is that in politics any preconceived plan –
> especially one that assumes that the millions of individuals composing a
> society will just automatically go along with it – is potentially
> dangerous. This is because the 'spirit of system' infects politicians
> with a messianic moral certainty that their reforms are so necessary and
> justified that almost any price is worth paying to achieve them.
> ... Thatcher's violent restructuring of the British economy during the
> 1980s was as much a product of the 'spirit of system' as any piece of
> top-down Soviet industrial strategy.

In short, Smith believed that politicians must spend their time limiting
the exploitation of the masses by economic elites, and leave any notion of
knowing how best to structure the economy at home.


text:
package org.apache.sentry.tests.e2e.hdfs;

import java.io.BufferedReader;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStreamReader;
import java.io.OutputStream;
import java.net.ServerSocket;
import java.net.URL;
import java.security.PrivilegedExceptionAction;
import java.sql.Connection;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.Statement;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.StringTokenizer;
import java.util.concurrent.atomic.AtomicBoolean;

import com.google.common.base.Preconditions;

import junit.framework.Assert;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.permission.AclEntry;
import org.apache.hadoop.fs.permission.AclEntryType;
import org.apache.hadoop.fs.permission.AclStatus;
import org.apache.hadoop.fs.permission.FsAction;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.hdfs.DFSTestUtil;
import org.apache.hadoop.hdfs.HdfsConfiguration;
import org.apache.hadoop.hdfs.MiniDFSCluster;
import org.apache.hadoop.hdfs.server.namenode.EditLogFileOutputStream;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapred.FileInputFormat;
import org.apache.hadoop.mapred.FileOutputFormat;
import org.apache.hadoop.mapred.JobClient;
import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.mapred.MapReduceBase;
import org.apache.hadoop.mapred.Mapper;
import org.apache.hadoop.mapred.MiniMRClientCluster;
import org.apache.hadoop.mapred.OutputCollector;
import org.apache.hadoop.mapred.Reducer;
import org.apache.hadoop.mapred.Reporter;
import org.apache.hadoop.mapred.RunningJob;
import org.apache.hadoop.mapred.TextInputFormat;
import org.apache.hadoop.mapred.TextOutputFormat;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.sentry.binding.hive.SentryHiveAuthorizationTaskFactoryImpl;
import org.apache.sentry.binding.hive.conf.HiveAuthzConf;
import org.apache.sentry.binding.hive.v2.metastore.MetastoreAuthzBindingV2;
import org.apache.sentry.hdfs.SentryAuthorizationConstants;
import org.apache.sentry.hdfs.SentryAuthorizationProvider;
import org.apache.sentry.provider.db.SimpleDBProviderBackend;
import org.apache.sentry.provider.file.LocalGroupResourceAuthorizationProvider;
import org.apache.sentry.provider.file.PolicyFile;
import org.apache.sentry.service.thrift.ServiceConstants.ServerConfig;
import org.apache.sentry.tests.e2e.hive.StaticUserGroup;
import org.apache.sentry.tests.e2e.hive.fs.MiniDFS;
import org.apache.sentry.tests.e2e.hive.hiveserver.HiveServerFactory;
import org.apache.sentry.tests.e2e.hive.hiveserver.InternalHiveServer;
import org.apache.sentry.tests.e2e.hive.hiveserver.InternalMetastoreServer;
import org.apache.sentry.tests.e2e.minisentry.SentrySrv;
import org.apache.sentry.tests.e2e.minisentry.SentrySrvFactory;
import org.fest.reflect.core.Reflection;
import org.junit.After;
import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.junit.Ignore;
import org.junit.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import com.google.common.collect.Maps;
import com.google.common.io.Files;
import com.google.common.io.Resources;

public class TestHDFSIntegration {

  private static final Logger LOGGER = LoggerFactory
      .getLogger(TestHDFSIntegration.class);

  public static class WordCountMapper extends MapReduceBase implements
      Mapper<LongWritable, Text, String, Long> {

    public void map(LongWritable key, Text value,
        OutputCollector<String, Long> output, Reporter reporter)
        throws IOException {
      StringTokenizer st = new StringTokenizer(value.toString());
      while (st.hasMoreTokens()) {
        output.collect(st.nextToken(), 1L);
      }
    }

  }

  public static class SumReducer extends MapReduceBase implements
      Reducer<Text, Long, Text, Long> {

    public void reduce(Text key, Iterator<Long> values,
        OutputCollector<Text, Long> output, Reporter reporter)
        throws IOException {

      long sum = 0;
      while (values.hasNext()) {
        sum += values.next();
      }
      output.collect(key, sum);
    }

  }

  private static final int NUM_RETRIES = 10;
  private static final int RETRY_WAIT = 1000;
  private static final String DFS_NAMENODE_AUTHORIZATION_PROVIDER_KEY =
      "dfs.namenode.authorization.provider.class";

  private static MiniDFSCluster miniDFS;
  private MiniMRClientCluster miniMR;
  private static InternalHiveServer hiveServer2;
  private static InternalMetastoreServer metastore;

  private static int sentryPort = -1;
  protected static SentrySrv sentryServer;
  protected static boolean testSentryHA = false;
  private static final long STALE_THRESHOLD = 5000;
  private static final long CACHE_REFRESH = 100; //Default is 500, but we want it to be low
                                                // in our tests so that changes reflect soon

  private static String fsURI;
  private static int hmsPort;
  private static File baseDir;
  private static File policyFileLocation;
  private static UserGroupInformation adminUgi;
  private static UserGroupInformation hiveUgi;

  // Variables which are used for cleanup after test
  // Please set these values in each test
  private Path tmpHDFSDir;
  private String[] dbNames;
  private String[] roles;
  private String admin;

  private static Configuration hadoopConf;

  protected static File assertCreateDir(File dir) {
    if(!dir.isDirectory()) {
      Assert.assertTrue("Failed creating " + dir, dir.mkdirs());
    }
    return dir;
  }

  private static int findPort() throws IOException {
    ServerSocket socket = new ServerSocket(0);
    int port = socket.getLocalPort();
    socket.close();
    return port;
  }

  @BeforeClass
  public static void setup() throws Exception {
    Class.forName("org.apache.hive.jdbc.HiveDriver");
    baseDir = Files.createTempDir();
    policyFileLocation = new File(baseDir, HiveServerFactory.AUTHZ_PROVIDER_FILENAME);
    PolicyFile policyFile = PolicyFile.setAdminOnServer1("hive")
        .setUserGroupMapping(StaticUserGroup.getStaticMapping());
    policyFile.write(policyFileLocation);

    adminUgi = UserGroupInformation.createUserForTesting(
        System.getProperty("user.name"), new String[] { "supergroup" });

    hiveUgi = UserGroupInformation.createUserForTesting(
        "hive", new String[] { "hive" });

    // Start Sentry
    startSentry();

    // Start HDFS and MR
    startDFSandYARN();

    // Start HiveServer2 and Metastore
    startHiveAndMetastore();

  }

  private static void startHiveAndMetastore() throws IOException, InterruptedException {
    startHiveAndMetastore(NUM_RETRIES);
  }

  private static void startHiveAndMetastore(final int retries) throws IOException, InterruptedException {
    hiveUgi.doAs(new PrivilegedExceptionAction<Void>() {
      @Override
      public Void run() throws Exception {
        HiveConf hiveConf = new HiveConf();
        hiveConf.set("sentry.metastore.plugins", "org.apache.sentry.hdfs.MetastorePlugin");
        hiveConf.set("sentry.service.client.server.rpc-address", "localhost");
        hiveConf.set("sentry.hdfs.service.client.server.rpc-address", "localhost");
        hiveConf.set("sentry.hdfs.service.client.server.rpc-port", String.valueOf(sentryPort));
        hiveConf.set("sentry.service.client.server.rpc-port", String.valueOf(sentryPort));
//        hiveConf.set("sentry.service.server.compact.transport", "true");
//        hiveConf.set("sentry.service.client.compact.transport", "true");
        hiveConf.set("sentry.service.security.mode", "none");
        hiveConf.set("sentry.hdfs.service.security.mode", "none");
        hiveConf.set("sentry.hdfs.init.update.retry.delay.ms", "500");
        hiveConf.set("sentry.hive.provider.backend", "org.apache.sentry.provider.db.SimpleDBProviderBackend");
        hiveConf.set("sentry.provider", LocalGroupResourceAuthorizationProvider.class.getName());
        hiveConf.set("sentry.hive.provider", LocalGroupResourceAuthorizationProvider.class.getName());
        hiveConf.set("sentry.hive.provider.resource", policyFileLocation.getPath());
        hiveConf.set("sentry.hive.testing.mode", "true");
        hiveConf.set("sentry.hive.server", "server1");

        hiveConf.set(ServerConfig.SENTRY_STORE_GROUP_MAPPING, ServerConfig.SENTRY_STORE_LOCAL_GROUP_MAPPING);
        hiveConf.set(ServerConfig.SENTRY_STORE_GROUP_MAPPING_RESOURCE, policyFileLocation.getPath());
        hiveConf.set("fs.defaultFS", fsURI);
        hiveConf.set("fs.default.name", fsURI);
        hiveConf.set("hive.metastore.execute.setugi", "true");
        hiveConf.set("hive.metastore.warehouse.dir", "hdfs:///user/hive/warehouse");
        hiveConf.set("javax.jdo.option.ConnectionURL", "jdbc:derby:;databaseName=" + baseDir.getAbsolutePath() + "/metastore_db;create=true");
        hiveConf.set("javax.jdo.option.ConnectionDriverName", "org.apache.derby.jdbc.EmbeddedDriver");
        hiveConf.set("javax.jdo.option.ConnectionUserName", "hive");
        hiveConf.set("javax.jdo.option.ConnectionPassword", "hive");
        hiveConf.set("datanucleus.autoCreateSchema", "true");
        hiveConf.set("datanucleus.fixedDatastore", "false");
        hiveConf.set("datanucleus.autoStartMechanism", "SchemaTable");
        hmsPort = findPort();
        LOGGER.info("\n\n HMS port : " + hmsPort + "\n\n");
        hiveConf.set("hive.metastore.uris", "thrift://localhost:" + hmsPort);
        hiveConf.set("hive.metastore.pre.event.listeners", MetastoreAuthzBindingV2.class.getName());
        hiveConf.set("hive.metastore.event.listeners", "org.apache.sentry.binding.metastore.SentryMetastorePostEventListener");
        hiveConf.set("hive.security.authorization.task.factory", "org.apache.sentry.binding.hive.SentryHiveAuthorizationTaskFactoryImpl");
        hiveConf.set("hive.server2.session.hook", "org.apache.sentry.binding.hive.HiveAuthzBindingSessionHook");
        hiveConf.set("sentry.metastore.service.users", "hive");// queries made by hive user (beeline) skip meta store check

        HiveAuthzConf authzConf = new HiveAuthzConf(Resources.getResource("sentry-site.xml"));
        authzConf.addResource(hiveConf);
        File confDir = assertCreateDir(new File(baseDir, "etc"));
        File accessSite = new File(confDir, HiveAuthzConf.AUTHZ_SITE_FILE);
        OutputStream out = new FileOutputStream(accessSite);
        authzConf.set("fs.defaultFS", fsURI);
        authzConf.writeXml(out);
        out.close();

        hiveConf.set("hive.sentry.conf.url", accessSite.getPath());
        LOGGER.info("Sentry client file : " + accessSite.getPath());

        File hiveSite = new File(confDir, "hive-site.xml");
        hiveConf.set("hive.server2.enable.doAs", "false");
        hiveConf.set(HiveAuthzConf.HIVE_SENTRY_CONF_URL, accessSite.toURI().toURL()
            .toExternalForm());
        out = new FileOutputStream(hiveSite);
        hiveConf.writeXml(out);
        out.close();

        Reflection.staticField("hiveSiteURL")
            .ofType(URL.class)
            .in(HiveConf.class)
            .set(hiveSite.toURI().toURL());

        metastore = new InternalMetastoreServer(hiveConf);
        new Thread() {
          @Override
          public void run() {
            try {
              metastore.start();
              while (true) {
              }
            } catch (Exception e) {
              LOGGER.info("Could not start Hive Server");
            }
          }
        }.start();

        startHiveServer2(retries, hiveConf);
        return null;
      }
    });
  }

  private static void startHiveServer2(final int retries, HiveConf hiveConf)
      throws IOException, InterruptedException, SQLException {
    Connection conn = null;
    Thread th = null;
    final AtomicBoolean keepRunning = new AtomicBoolean(true);
    try {
      hiveServer2 = new InternalHiveServer(hiveConf);
      th = new Thread() {
        @Override
        public void run() {
          try {
            hiveServer2.start();
            while(keepRunning.get()){}
          } catch (Exception e) {
            LOGGER.info("Could not start Hive Server");
          }
        }
      };
      th.start();
      Thread.sleep(RETRY_WAIT * 5);
      conn = hiveServer2.createConnection("hive", "hive");
    } catch (Exception ex) {
      if (retries > 0) {
        try {
          keepRunning.set(false);
          hiveServer2.shutdown();
        } catch (Exception e) {
          // Ignore
        }
        LOGGER.info("Re-starting Hive Server2 !!");
        startHiveServer2(retries - 1, hiveConf);
      }
    }
    if (conn != null) {
      conn.close();
    }
  }

  private static String getSentryPort() throws Exception{
    if(sentryServer!=null) {
      return String.valueOf(sentryServer.get(0).getAddress().getPort());
    } else {
      throw new Exception("Sentry server not initialized");
    }
  }
  private static void startDFSandYARN() throws IOException,
      InterruptedException {
    adminUgi.doAs(new PrivilegedExceptionAction<Void>() {
      @Override
      public Void run() throws Exception {
        System.setProperty(MiniDFSCluster.PROP_TEST_BUILD_DATA, "target/test/data");
        hadoopConf = new HdfsConfiguration();
        hadoopConf.set(DFS_NAMENODE_AUTHORIZATION_PROVIDER_KEY,
            SentryAuthorizationProvider.class.getName());
        hadoopConf.setBoolean(DFSConfigKeys.DFS_NAMENODE_ACLS_ENABLED_KEY, true);
        hadoopConf.setInt(DFSConfigKeys.DFS_REPLICATION_KEY, 1);
        File dfsDir = assertCreateDir(new File(baseDir, "dfs"));
        hadoopConf.set(MiniDFSCluster.HDFS_MINIDFS_BASEDIR, dfsDir.getPath());
        hadoopConf.set("hadoop.security.group.mapping",
            MiniDFS.PseudoGroupMappingService.class.getName());
        Configuration.addDefaultResource("test.xml");

        hadoopConf.set("sentry.authorization-provider.hdfs-path-prefixes", "/user/hive/warehouse,/tmp/external");
        hadoopConf.set("sentry.authorization-provider.cache-refresh-retry-wait.ms", "5000");
        hadoopConf.set("sentry.authorization-provider.cache-refresh-interval.ms", String.valueOf(CACHE_REFRESH));

        hadoopConf.set("sentry.authorization-provider.cache-stale-threshold.ms", String.valueOf(STALE_THRESHOLD));

        hadoopConf.set("sentry.hdfs.service.security.mode", "none");
        hadoopConf.set("sentry.hdfs.service.client.server.rpc-address", "localhost");
        hadoopConf.set("sentry.hdfs.service.client.server.rpc-port", String.valueOf(sentryPort));
        EditLogFileOutputStream.setShouldSkipFsyncForTesting(true);
        miniDFS = new MiniDFSCluster.Builder(hadoopConf).build();
        Path tmpPath = new Path("/tmp");
        Path hivePath = new Path("/user/hive");
        Path warehousePath = new Path(hivePath, "warehouse");
        miniDFS.getFileSystem().mkdirs(warehousePath);
        boolean directory = miniDFS.getFileSystem().isDirectory(warehousePath);
        LOGGER.info("\n\n Is dir :" + directory + "\n\n");
        LOGGER.info("\n\n DefaultFS :" + miniDFS.getFileSystem().getUri() + "\n\n");
        fsURI = miniDFS.getFileSystem().getUri().toString();
        hadoopConf.set("fs.defaultFS", fsURI);

        // Create Yarn cluster
        // miniMR = MiniMRClientClusterFactory.create(this.getClass(), 1, conf);

        miniDFS.getFileSystem().mkdirs(tmpPath);
        miniDFS.getFileSystem().setPermission(tmpPath, FsPermission.valueOf("drwxrwxrwx"));
        miniDFS.getFileSystem().setOwner(hivePath, "hive", "hive");
        miniDFS.getFileSystem().setOwner(warehousePath, "hive", "hive");
        LOGGER.info("\n\n Owner :"
            + miniDFS.getFileSystem().getFileStatus(warehousePath).getOwner()
            + ", "
            + miniDFS.getFileSystem().getFileStatus(warehousePath).getGroup()
            + "\n\n");
        LOGGER.info("\n\n Owner tmp :"
            + miniDFS.getFileSystem().getFileStatus(tmpPath).getOwner() + ", "
            + miniDFS.getFileSystem().getFileStatus(tmpPath).getGroup() + ", "
            + miniDFS.getFileSystem().getFileStatus(tmpPath).getPermission() + ", "
            + "\n\n");

        int dfsSafeCheckRetry = 30;
        boolean hasStarted = false;
        for (int i = dfsSafeCheckRetry; i > 0; i--) {
          if (!miniDFS.getFileSystem().isInSafeMode()) {
            hasStarted = true;
            LOGGER.info("HDFS safemode check num times : " + (31 - i));
            break;
          }
        }
        if (!hasStarted) {
          throw new RuntimeException("HDFS hasnt exited safe mode yet..");
        }

        return null;
      }
    });
  }

  private static void startSentry() throws Exception {
    try {

      hiveUgi.doAs(new PrivilegedExceptionAction<Void>() {
        @Override
        public Void run() throws Exception {
          Configuration sentryConf = new Configuration(false);
          Map<String, String> properties = Maps.newHashMap();
          properties.put(HiveServerFactory.AUTHZ_PROVIDER_BACKEND,
              SimpleDBProviderBackend.class.getName());
          properties.put(ConfVars.HIVE_AUTHORIZATION_TASK_FACTORY.varname,
              SentryHiveAuthorizationTaskFactoryImpl.class.getName());
          properties
              .put(ConfVars.HIVE_SERVER2_THRIFT_MIN_WORKER_THREADS.varname, "2");
          properties.put("hive.metastore.uris", "thrift://localhost:" + hmsPort);
          properties.put("hive.exec.local.scratchdir", Files.createTempDir().getAbsolutePath());
          properties.put(ServerConfig.SECURITY_MODE, ServerConfig.SECURITY_MODE_NONE);
//        properties.put("sentry.service.server.compact.transport", "true");
          properties.put("sentry.hive.testing.mode", "true");
          properties.put("sentry.service.reporting", "JMX");
          properties.put(ServerConfig.ADMIN_GROUPS, "hive,admin");
          properties.put(ServerConfig.RPC_ADDRESS, "localhost");
          properties.put(ServerConfig.RPC_PORT, String.valueOf(sentryPort > 0 ? sentryPort : 0));
          properties.put(ServerConfig.SENTRY_VERIFY_SCHEM_VERSION, "false");

          properties.put(ServerConfig.SENTRY_STORE_GROUP_MAPPING, ServerConfig.SENTRY_STORE_LOCAL_GROUP_MAPPING);
          properties.put(ServerConfig.SENTRY_STORE_GROUP_MAPPING_RESOURCE, policyFileLocation.getPath());
          properties.put(ServerConfig.SENTRY_STORE_JDBC_URL,
              "jdbc:derby:;databaseName=" + baseDir.getPath()
                  + "/sentrystore_db;create=true");
          properties.put(ServerConfig.SENTRY_STORE_JDBC_PASS, "dummy");
          properties.put("sentry.service.processor.factories",
              "org.apache.sentry.provider.db.service.thrift.SentryPolicyStoreProcessorFactory,org.apache.sentry.hdfs.SentryHDFSServiceProcessorFactory");
          properties.put("sentry.policy.store.plugins", "org.apache.sentry.hdfs.SentryPlugin");
          properties.put(ServerConfig.RPC_MIN_THREADS, "3");
          for (Map.Entry<String, String> entry : properties.entrySet()) {
            sentryConf.set(entry.getKey(), entry.getValue());
          }
          sentryServer = SentrySrvFactory.create(SentrySrvFactory.SentrySrvType.INTERNAL_SERVER,
              sentryConf, testSentryHA ? 2 : 1);
          sentryPort = sentryServer.get(0).getAddress().getPort();
          sentryServer.startAll();
          LOGGER.info("\n\n Sentry service started \n\n");
          return null;
        }
      });
    } catch (Exception e) {
      //An exception happening in above block will result in a wrapped UndeclaredThrowableException.
      throw new Exception(e.getCause());
    }
  }

  @After
  public void cleanAfterTest() throws Exception {
    //Clean up database
    Connection conn;
    Statement stmt;
    Preconditions.checkArgument(admin != null && dbNames !=null && roles != null && tmpHDFSDir != null,
        "Test case did not set some of these values required for clean up: admin, dbNames, roles, tmpHDFSDir");

    conn = hiveServer2.createConnection(admin, admin);
    stmt = conn.createStatement();
    for( String dbName: dbNames) {
      stmt.execute("drop database if exists " + dbName + " cascade");
    }
    stmt.close();
    conn.close();

    //Clean up roles
    conn = hiveServer2.createConnection("hive", "hive");
    stmt = conn.createStatement();
    for( String role:roles) {
       stmt.execute("drop role " + role);
    }
    stmt.close();
    conn.close();

    //Clean up hdfs directories
    miniDFS.getFileSystem().delete(tmpHDFSDir, true);

    tmpHDFSDir = null;
    dbNames = null;
    roles = null;
    admin = null;
  }

  @AfterClass
  public static void cleanUp() throws Exception {
    try {
      if (miniDFS != null) {
        miniDFS.shutdown();
      }
    } finally {
      try {
        if (hiveServer2 != null) {
          hiveServer2.shutdown();
        }
      } finally {
        try {
          if (metastore != null) {
            metastore.shutdown();
          }
        } finally {
          sentryServer.close();
        }
      }
    }
  }

  @Test
  public void testEnd2End() throws Throwable {
    tmpHDFSDir = new Path("/tmp/external");
    dbNames = new String[]{"db1"};
    roles = new String[]{"admin_role", "db_role", "tab_role", "p1_admin"};
    admin = "hive";

    Connection conn;
    Statement stmt;
    conn = hiveServer2.createConnection("hive", "hive");
    stmt = conn.createStatement();
    stmt.execute("create role admin_role");
    stmt.execute("grant role admin_role to group hive");
    stmt.execute("grant all on server server1 to role admin_role");
    stmt.execute("create table p1 (s string) partitioned by (month int, day int)");
    stmt.execute("alter table p1 add partition (month=1, day=1)");
    stmt.execute("alter table p1 add partition (month=1, day=2)");
    stmt.execute("alter table p1 add partition (month=2, day=1)");
    stmt.execute("alter table p1 add partition (month=2, day=2)");

    // db privileges
    stmt.execute("create database db5");
    stmt.execute("create role db_role");
    stmt.execute("create role tab_role");
    stmt.execute("grant role db_role to group hbase");
    stmt.execute("grant role tab_role to group flume");
    stmt.execute("create table db5.p2(id int)");

    stmt.execute("create role p1_admin");
    stmt.execute("grant role p1_admin to group hbase");

    // Verify default db is inaccessible initially
    verifyOnAllSubDirs("/user/hive/warehouse", null, "hbase", false);

    verifyOnAllSubDirs("/user/hive/warehouse/p1", null, "hbase", false);

    stmt.execute("grant all on database db5 to role db_role");
    stmt.execute("use db5");
    stmt.execute("grant all on table p2 to role tab_role");
    stmt.execute("use default");
    verifyOnAllSubDirs("/user/hive/warehouse/db5.db", FsAction.ALL, "hbase", true);
    verifyOnAllSubDirs("/user/hive/warehouse/db5.db/p2", FsAction.ALL, "hbase", true);
    verifyOnAllSubDirs("/user/hive/warehouse/db5.db/p2", FsAction.ALL, "flume", true);
    verifyOnPath("/user/hive/warehouse/db5.db", FsAction.ALL, "flume", false);

    loadData(stmt);

    verifyHDFSandMR(stmt);

    // Verify default db is STILL inaccessible after grants but tables are fine
    verifyOnPath("/user/hive/warehouse", null, "hbase", false);
    verifyOnAllSubDirs("/user/hive/warehouse/p1", FsAction.READ_EXECUTE, "hbase", true);

    adminUgi.doAs(new PrivilegedExceptionAction<Void>() {
      @Override
      public Void run() throws Exception {
        // Simulate hdfs dfs -setfacl -m <aclantry> <path>
        AclStatus existing =
            miniDFS.getFileSystem()
            .getAclStatus(new Path("/user/hive/warehouse/p1"));
        ArrayList<AclEntry> newEntries =
            new ArrayList<AclEntry>(existing.getEntries());
        newEntries.add(AclEntry.parseAclEntry("user::---", true));
        newEntries.add(AclEntry.parseAclEntry("group:bla:rwx", true));
        newEntries.add(AclEntry.parseAclEntry("other::---", true));
        miniDFS.getFileSystem().setAcl(new Path("/user/hive/warehouse/p1"),
            newEntries);
        return null;
      }
    });

    stmt.execute("revoke select on table p1 from role p1_admin");
    verifyOnAllSubDirs("/user/hive/warehouse/p1", null, "hbase", false);

    // Verify default db grants work
    stmt.execute("grant select on database default to role p1_admin");
    verifyOnPath("/user/hive/warehouse", FsAction.READ_EXECUTE, "hbase", true);

    // Verify default db grants are propagated to the tables
    verifyOnAllSubDirs("/user/hive/warehouse/p1", FsAction.READ_EXECUTE, "hbase", true);

    // Verify default db revokes work
    stmt.execute("revoke select on database default from role p1_admin");
    verifyOnPath("/user/hive/warehouse", null, "hbase", false);
    verifyOnAllSubDirs("/user/hive/warehouse/p1", null, "hbase", false);

    stmt.execute("grant all on table p1 to role p1_admin");
    verifyOnAllSubDirs("/user/hive/warehouse/p1", FsAction.ALL, "hbase", true);

    stmt.execute("revoke select on table p1 from role p1_admin");
    verifyOnAllSubDirs("/user/hive/warehouse/p1", FsAction.WRITE_EXECUTE, "hbase", true);

    // Verify table rename works
    stmt.execute("alter table p1 rename to p3");
    verifyOnAllSubDirs("/user/hive/warehouse/p3", FsAction.WRITE_EXECUTE, "hbase", true);

    stmt.execute("alter table p3 partition (month=1, day=1) rename to partition (month=1, day=3)");
    verifyOnAllSubDirs("/user/hive/warehouse/p3", FsAction.WRITE_EXECUTE, "hbase", true);
    verifyOnAllSubDirs("/user/hive/warehouse/p3/month=1/day=3", FsAction.WRITE_EXECUTE, "hbase", true);

    // Test DB case insensitivity
    stmt.execute("create database extdb");
    stmt.execute("grant all on database ExtDb to role p1_admin");
    writeToPath("/tmp/external/ext100", 5, "foo", "bar");
    writeToPath("/tmp/external/ext101", 5, "foo", "bar");
    stmt.execute("use extdb");
    stmt.execute(
            "create table ext100 (s string) location \'/tmp/external/ext100\'");
    verifyQuery(stmt, "ext100", 5);
    verifyOnAllSubDirs("/tmp/external/ext100", FsAction.ALL, "hbase", true);
    stmt.execute("use default");

    stmt.execute("use EXTDB");
    stmt.execute(
            "create table ext101 (s string) location \'/tmp/external/ext101\'");
    verifyQuery(stmt, "ext101", 5);
    verifyOnAllSubDirs("/tmp/external/ext101", FsAction.ALL, "hbase", true);

    // Test table case insensitivity
    stmt.execute("grant all on table exT100 to role tab_role");
    verifyOnAllSubDirs("/tmp/external/ext100", FsAction.ALL, "flume", true);

    stmt.execute("use default");

    //TODO: SENTRY-795: HDFS permissions do not sync when Sentry restarts in HA mode.
    if(!testSentryHA) {
      long beforeStop = System.currentTimeMillis();
      sentryServer.stopAll();
      long timeTakenForStopMs = System.currentTimeMillis() - beforeStop;
      LOGGER.info("Time taken for Sentry server stop: " + timeTakenForStopMs);

      // Verify that Sentry permission are still enforced for the "stale" period only if stop did not take too long
      if(timeTakenForStopMs < STALE_THRESHOLD) {
        verifyOnAllSubDirs("/user/hive/warehouse/p3", FsAction.WRITE_EXECUTE, "hbase", true);
        Thread.sleep((STALE_THRESHOLD - timeTakenForStopMs));
      } else {
        LOGGER.warn("Sentry server stop took too long");
      }

      // Verify that Sentry permission are NOT enforced AFTER "stale" period
      verifyOnAllSubDirs("/user/hive/warehouse/p3", null, "hbase", false);

      sentryServer.startAll();
    }

    // Verify that After Sentry restart permissions are re-enforced
    verifyOnAllSubDirs("/user/hive/warehouse/p3", FsAction.WRITE_EXECUTE, "hbase", true);

    // Create new table and verify everything is fine after restart...
    stmt.execute("create table p2 (s string) partitioned by (month int, day int)");
    stmt.execute("alter table p2 add partition (month=1, day=1)");
    stmt.execute("alter table p2 add partition (month=1, day=2)");
    stmt.execute("alter table p2 add partition (month=2, day=1)");
    stmt.execute("alter table p2 add partition (month=2, day=2)");

    verifyOnAllSubDirs("/user/hive/warehouse/p2", null, "hbase", false);

    stmt.execute("grant select on table p2 to role p1_admin");
    verifyOnAllSubDirs("/user/hive/warehouse/p2", FsAction.READ_EXECUTE, "hbase", true);

    stmt.execute("grant select on table p2 to role p1_admin");
    verifyOnAllSubDirs("/user/hive/warehouse/p2", FsAction.READ_EXECUTE, "hbase", true);

    // Create external table
    writeToPath("/tmp/external/ext1", 5, "foo", "bar");

    stmt.execute("create table ext1 (s string) location \'/tmp/external/ext1\'");
    verifyQuery(stmt, "ext1", 5);

    // Ensure existing group permissions are never returned..
    verifyOnAllSubDirs("/tmp/external/ext1", null, "bar", false);
    verifyOnAllSubDirs("/tmp/external/ext1", null, "hbase", false);

    stmt.execute("grant all on table ext1 to role p1_admin");
    verifyOnAllSubDirs("/tmp/external/ext1", FsAction.ALL, "hbase", true);

    stmt.execute("revoke select on table ext1 from role p1_admin");
    verifyOnAllSubDirs("/tmp/external/ext1", FsAction.WRITE_EXECUTE, "hbase", true);

    // Verify database operations works correctly
    stmt.execute("create database db1");
    verifyOnAllSubDirs("/user/hive/warehouse/db1.db", null, "hbase", false);

    stmt.execute("create table db1.tbl1 (s string)");
    verifyOnAllSubDirs("/user/hive/warehouse/db1.db/tbl1", null, "hbase", false);
    stmt.execute("create table db1.tbl2 (s string)");
    verifyOnAllSubDirs("/user/hive/warehouse/db1.db/tbl2", null, "hbase", false);

    // Verify default db grants do not affect other dbs
    stmt.execute("grant all on database default to role p1_admin");
    verifyOnPath("/user/hive/warehouse", FsAction.ALL, "hbase", true);
    verifyOnAllSubDirs("/user/hive/warehouse/db1.db", null, "hbase", false);

    // Verify table rename works
    stmt.execute("create table q1 (s string)");
    verifyOnAllSubDirs("/user/hive/warehouse/q1", FsAction.ALL, "hbase", true);
    stmt.execute("alter table q1 rename to q2");
    verifyOnAllSubDirs("/user/hive/warehouse/q2", FsAction.ALL, "hbase", true);

    // Verify table GRANTS do not trump db GRANTS
    stmt.execute("grant select on table q2 to role p1_admin");
    verifyOnAllSubDirs("/user/hive/warehouse/q2", FsAction.ALL, "hbase", true);

    stmt.execute("create table q3 (s string)");
    verifyOnAllSubDirs("/user/hive/warehouse/q3", FsAction.ALL, "hbase", true);
    verifyOnAllSubDirs("/user/hive/warehouse/q2", FsAction.ALL, "hbase", true);

    // Verify db privileges are propagated to tables
    stmt.execute("grant select on database db1 to role p1_admin");
    verifyOnAllSubDirs("/user/hive/warehouse/db1.db/tbl1", FsAction.READ_EXECUTE, "hbase", true);
    verifyOnAllSubDirs("/user/hive/warehouse/db1.db/tbl2", FsAction.READ_EXECUTE, "hbase", true);

    // Verify default db revokes do not affect other dbs
    stmt.execute("revoke all on database default from role p1_admin");
    verifyOnPath("/user/hive/warehouse", null, "hbase", false);
    verifyOnAllSubDirs("/user/hive/warehouse/db1.db/tbl1", FsAction.READ_EXECUTE, "hbase", true);
    verifyOnAllSubDirs("/user/hive/warehouse/db1.db/tbl2", FsAction.READ_EXECUTE, "hbase", true);

    stmt.execute("use db1");
    stmt.execute("grant all on table tbl1 to role p1_admin");

    verifyOnAllSubDirs("/user/hive/warehouse/db1.db/tbl1", FsAction.ALL, "hbase", true);
    verifyOnAllSubDirs("/user/hive/warehouse/db1.db/tbl2", FsAction.READ_EXECUTE, "hbase", true);

    // Verify recursive revoke
    stmt.execute("revoke select on database db1 from role p1_admin");

    verifyOnAllSubDirs("/user/hive/warehouse/db1.db/tbl1", FsAction.WRITE_EXECUTE, "hbase", true);
    verifyOnAllSubDirs("/user/hive/warehouse/db1.db/tbl2", null, "hbase", false);

    // Verify cleanup..
    stmt.execute("drop table tbl1");
    Assert.assertFalse(miniDFS.getFileSystem().exists(new Path("/user/hive/warehouse/db1.db/tbl1")));

    stmt.execute("drop table tbl2");
    Assert.assertFalse(miniDFS.getFileSystem().exists(new Path("/user/hive/warehouse/db1.db/tbl2")));

    stmt.execute("use default");
    stmt.execute("drop database db1");
    Assert.assertFalse(miniDFS.getFileSystem().exists(new Path("/user/hive/warehouse/db1.db")));

    // START : Verify external table set location..
    writeToPath("/tmp/external/tables/ext2_before/i=1", 5, "foo", "bar");
    writeToPath("/tmp/external/tables/ext2_before/i=2", 5, "foo", "bar");

    stmt.execute("create external table ext2 (s string) partitioned by (i int) location \'/tmp/external/tables/ext2_before\'");
    stmt.execute("alter table ext2 add partition (i=1)");
    stmt.execute("alter table ext2 add partition (i=2)");
    verifyQuery(stmt, "ext2", 10);
    verifyOnAllSubDirs("/tmp/external/tables/ext2_before", null, "hbase", false);
    stmt.execute("grant all on table ext2 to role p1_admin");
    verifyOnPath("/tmp/external/tables/ext2_before", FsAction.ALL, "hbase", true);
    verifyOnPath("/tmp/external/tables/ext2_before/i=1", FsAction.ALL, "hbase", true);
    verifyOnPath("/tmp/external/tables/ext2_before/i=2", FsAction.ALL, "hbase", true);
    verifyOnPath("/tmp/external/tables/ext2_before/i=1/stuff.txt", FsAction.ALL, "hbase", true);
    verifyOnPath("/tmp/external/tables/ext2_before/i=2/stuff.txt", FsAction.ALL, "hbase", true);

    writeToPath("/tmp/external/tables/ext2_after/i=1", 6, "foo", "bar");
    writeToPath("/tmp/external/tables/ext2_after/i=2", 6, "foo", "bar");

    stmt.execute("alter table ext2 set location \'hdfs:///tmp/external/tables/ext2_after\'");
    // Even though table location is altered, partition location is still old (still 10 rows)
    verifyQuery(stmt, "ext2", 10);
    // You have to explicitly alter partition location..
    verifyOnPath("/tmp/external/tables/ext2_before", null, "hbase", false);
    verifyOnPath("/tmp/external/tables/ext2_before/i=1", FsAction.ALL, "hbase", true);
    verifyOnPath("/tmp/external/tables/ext2_before/i=2", FsAction.ALL, "hbase", true);
    verifyOnPath("/tmp/external/tables/ext2_before/i=1/stuff.txt", FsAction.ALL, "hbase", true);
    verifyOnPath("/tmp/external/tables/ext2_before/i=2/stuff.txt", FsAction.ALL, "hbase", true);

    stmt.execute("alter table ext2 partition (i=1) set location \'hdfs:///tmp/external/tables/ext2_after/i=1\'");
    stmt.execute("alter table ext2 partition (i=2) set location \'hdfs:///tmp/external/tables/ext2_after/i=2\'");
    // Now that partition location is altered, it picks up new data (12 rows instead of 10)
    verifyQuery(stmt, "ext2", 12);

    verifyOnPath("/tmp/external/tables/ext2_before", null, "hbase", false);
    verifyOnPath("/tmp/external/tables/ext2_before/i=1", null, "hbase", false);
    verifyOnPath("/tmp/external/tables/ext2_before/i=2", null, "hbase", false);
    verifyOnPath("/tmp/external/tables/ext2_before/i=1/stuff.txt", null, "hbase", false);
    verifyOnPath("/tmp/external/tables/ext2_before/i=2/stuff.txt", null, "hbase", false);
    verifyOnPath("/tmp/external/tables/ext2_after", FsAction.ALL, "hbase", true);
    verifyOnPath("/tmp/external/tables/ext2_after/i=1", FsAction.ALL, "hbase", true);
    verifyOnPath("/tmp/external/tables/ext2_after/i=2", FsAction.ALL, "hbase", true);
    verifyOnPath("/tmp/external/tables/ext2_after/i=1/stuff.txt", FsAction.ALL, "hbase", true);
    verifyOnPath("/tmp/external/tables/ext2_after/i=2/stuff.txt", FsAction.ALL, "hbase", true);
    // END : Verify external table set location..

    // Restart HDFS to verify if things are fine after re-start..

    // TODO : this is currently commented out since miniDFS.restartNameNode() does
    //        not work corectly on the version of hadoop sentry depends on
    //        This has been verified to work on a real cluster.
    //        Once miniDFS is fixed, this should be uncommented..
    // miniDFS.shutdown();
    // miniDFS.restartNameNode(true);
    // miniDFS.waitActive();
    // verifyOnPath("/tmp/external/tables/ext2_after", FsAction.ALL, "hbase", true);
    // verifyOnAllSubDirs("/user/hive/warehouse/p2", FsAction.READ_EXECUTE, "hbase", true);

    stmt.close();
    conn.close();
  }

  /**
   * Make sure non HDFS paths are not added to the object - location map.
   * @throws Throwable
   */
  @Test
  public void testNonHDFSLocations() throws Throwable {
    String dbName = "db2";

    tmpHDFSDir = new Path("/tmp/external");
    dbNames = new String[]{dbName};
    roles = new String[]{"admin_role", "user_role"};
    admin = StaticUserGroup.ADMIN1;

    Connection conn;
    Statement stmt;

    conn = hiveServer2.createConnection("hive", "hive");
    stmt = conn.createStatement();
    stmt.execute("create role admin_role");
    stmt.execute("grant all on server server1 to role admin_role");
    stmt.execute("grant all on uri 'file:///tmp/external' to role admin_role");
    stmt.execute("grant all on uri 'hdfs:///tmp/external' to role admin_role");
    stmt.execute("grant role admin_role to group " + StaticUserGroup.ADMINGROUP);

    conn = hiveServer2.createConnection(admin, admin);
    stmt = conn.createStatement();
    stmt.execute("create database " + dbName);
    stmt.close();
    conn.close();

    conn = hiveServer2.createConnection("hive", "hive");
    stmt = conn.createStatement();
    stmt.execute("create role user_role");
    stmt.execute("grant all on database " + dbName + " to role user_role");
    stmt.execute("grant role user_role to group " + StaticUserGroup.USERGROUP1);
    stmt.close();
    conn.close();

    conn = hiveServer2.createConnection(admin, admin);
    stmt = conn.createStatement();

    miniDFS.getFileSystem().mkdirs(tmpHDFSDir);
    miniDFS.getFileSystem().setOwner(tmpHDFSDir, "hive", "hive");

    //External table on local file system
    miniDFS.getFileSystem().mkdirs(new Path("/tmp/external/tab1_loc"));
    stmt.execute("use " + dbName);
    stmt.execute("create external table tab1(a int) location 'file:///tmp/external/tab1_loc'");
    verifyOnAllSubDirs("/tmp/external/tab1_loc", null, StaticUserGroup.USERGROUP1, false);

    //External partitioned table on local file system
    miniDFS.getFileSystem().mkdirs(new Path("/tmp/external/tab2_loc/i=1"));
    stmt.execute("create external table tab2 (s string) partitioned by (i int) location 'file:///tmp/external/tab2_loc'");
    verifyOnAllSubDirs("/tmp/external/tab2_loc", null, StaticUserGroup.USERGROUP1, false);
    //Partition on local file system
    stmt.execute("alter table tab2 add partition (i=1)");
    stmt.execute("alter table tab2 partition (i=1) set location 'file:///tmp/external/tab2_loc/i=1'");
    verifyOnAllSubDirs("/tmp/external/tab2_loc/i=1", null, StaticUserGroup.USERGROUP1, false);

    //HDFS to local file system, also make sure does not specifying scheme still works
    stmt.execute("create external table tab3(a int) location '/tmp/external/tab3_loc'");
    // SENTRY-546
    // verifyOnAllSubDirs("/tmp/external/tab3_loc", FsAction.ALL, StaticUserGroup.USERGROUP1, true);
    verifyOnAllSubDirs("/tmp/external/tab3_loc", null, StaticUserGroup.USERGROUP1, true);
    stmt.execute("alter table tab3 set location 'file:///tmp/external/tab3_loc'");
    verifyOnAllSubDirs("/tmp/external/tab3_loc", null, StaticUserGroup.USERGROUP1, false);

    //Local file system to HDFS
    stmt.execute("create table tab4(a int) location 'file:///tmp/external/tab4_loc'");
    stmt.execute("alter table tab4 set location 'hdfs:///tmp/external/tab4_loc'");
    miniDFS.getFileSystem().mkdirs(new Path("/tmp/external/tab4_loc"));
    // SENTRY-546
    // verifyOnAllSubDirs("/tmp/external/tab4_loc", FsAction.ALL, StaticUserGroup.USERGROUP1, true);
    verifyOnAllSubDirs("/tmp/external/tab4_loc", null, StaticUserGroup.USERGROUP1, true);
    stmt.close();
    conn.close();
  }

  @Ignore("SENTRY-546")
  @Test
  public void testExternalTable() throws Throwable {
    String dbName = "db2";

    tmpHDFSDir = new Path("/tmp/external");
    dbNames = new String[]{dbName};
    roles = new String[]{"admin_role"};
    admin = StaticUserGroup.ADMIN1;

    Connection conn;
    Statement stmt;

    conn = hiveServer2.createConnection("hive", "hive");
    stmt = conn.createStatement();
    stmt.execute("create role admin_role");
    stmt.execute("grant all on server server1 to role admin_role");
    stmt.execute("grant all on uri 'hdfs:///tmp/external' to role admin_role");
    stmt.execute("grant role admin_role to group " + StaticUserGroup.ADMINGROUP);

    conn = hiveServer2.createConnection(StaticUserGroup.ADMIN1, StaticUserGroup.ADMIN1);
    stmt = conn.createStatement();
    stmt.execute("create database " + dbName);
    stmt.execute("create external table tab1(a int) location '/tmp/external/tab1_loc'");
    verifyOnAllSubDirs("/tmp/external/tab1_loc", FsAction.ALL, StaticUserGroup.ADMINGROUP, true);

    stmt.close();
    conn.close();

  }

  @Test
  public void testColumnPrivileges() throws Throwable {
    String dbName = "db2";

    tmpHDFSDir = new Path("/tmp/external");
    dbNames = new String[]{dbName};
    roles = new String[]{"admin_role", "tab_role", "db_role", "col_role"};
    admin = StaticUserGroup.ADMIN1;

    Connection conn;
    Statement stmt;

    conn = hiveServer2.createConnection("hive", "hive");
    stmt = conn.createStatement();
    stmt.execute("create role admin_role");
    stmt.execute("grant all on server server1 to role admin_role with grant option");
    stmt.execute("grant role admin_role to group " + StaticUserGroup.ADMINGROUP);

    conn = hiveServer2.createConnection(StaticUserGroup.ADMIN1, StaticUserGroup.ADMIN1);
    stmt = conn.createStatement();
    stmt.execute("create database " + dbName);
    stmt.execute("use "+ dbName);
    stmt.execute("create table p1 (s string) partitioned by (month int, day int)");
    stmt.execute("alter table p1 add partition (month=1, day=1)");
    stmt.execute("alter table p1 add partition (month=1, day=2)");
    stmt.execute("alter table p1 add partition (month=2, day=1)");
    stmt.execute("alter table p1 add partition (month=2, day=2)");
    loadData(stmt);

    stmt.execute("create role db_role");
    stmt.execute("grant select on database " + dbName + " to role db_role");
    stmt.execute("create role tab_role");
    stmt.execute("grant select on p1 to role tab_role");
    stmt.execute("create role col_role");
    stmt.execute("grant select(s) on p1 to role col_role");

    stmt.execute("grant role col_role to group "+ StaticUserGroup.USERGROUP1);

    stmt.execute("grant role tab_role to group "+ StaticUserGroup.USERGROUP2);
    stmt.execute("grant role col_role to group "+ StaticUserGroup.USERGROUP2);

    stmt.execute("grant role db_role to group "+ StaticUserGroup.USERGROUP3);
    stmt.execute("grant role col_role to group "+ StaticUserGroup.USERGROUP3);

    stmt.execute("grant role col_role to group " + StaticUserGroup.ADMINGROUP);

    Thread.sleep(CACHE_REFRESH);//Wait till sentry cache is updated in Namenode

    //User with just column level privileges cannot read HDFS
    verifyOnAllSubDirs("/user/hive/warehouse/" + dbName + ".db/p1", null, StaticUserGroup.USERGROUP1, false);

    //User with permissions on table and column can read HDFS file
    verifyOnAllSubDirs("/user/hive/warehouse/" + dbName + ".db/p1", FsAction.READ_EXECUTE, StaticUserGroup.USERGROUP2, true);

    //User with permissions on db and column can read HDFS file
    verifyOnAllSubDirs("/user/hive/warehouse/" + dbName + ".db/p1", FsAction.READ_EXECUTE, StaticUserGroup.USERGROUP3, true);

    //User with permissions on server and column cannot read HDFS file
    //TODO:SENTRY-751
    verifyOnAllSubDirs("/user/hive/warehouse/" + dbName + ".db/p1", null, StaticUserGroup.ADMINGROUP, false);

    stmt.close();
    conn.close();

  }

  /*
  TODO:SENTRY-819
  */
  @Test
  public void testAllColumn() throws Throwable {
    String dbName = "db2";

    tmpHDFSDir = new Path("/tmp/external");
    dbNames = new String[]{dbName};
    roles = new String[]{"admin_role", "col_role"};
    admin = StaticUserGroup.ADMIN1;

    Connection conn;
    Statement stmt;

    conn = hiveServer2.createConnection("hive", "hive");
    stmt = conn.createStatement();
    stmt.execute("create role admin_role");
    stmt.execute("grant all on server server1 to role admin_role with grant option");
    stmt.execute("grant role admin_role to group " + StaticUserGroup.ADMINGROUP);

    conn = hiveServer2.createConnection(StaticUserGroup.ADMIN1, StaticUserGroup.ADMIN1);
    stmt = conn.createStatement();
    stmt.execute("create database " + dbName);
    stmt.execute("use "+ dbName);
    stmt.execute("create table p1 (c1 string, c2 string) partitioned by (month int, day int)");
    stmt.execute("alter table p1 add partition (month=1, day=1)");
    loadDataTwoCols(stmt);

    stmt.execute("create role col_role");
    stmt.execute("grant select(c1,c2) on p1 to role col_role");
    stmt.execute("grant role col_role to group "+ StaticUserGroup.USERGROUP1);
    Thread.sleep(100);

    //User with privileges on all columns of the data cannot still read the HDFS files
    verifyOnAllSubDirs("/user/hive/warehouse/" + dbName + ".db/p1", null, StaticUserGroup.USERGROUP1, false);

    stmt.close();
    conn.close();

  }
  //SENTRY-780
  @Test
  public void testViews() throws Throwable {
    String dbName= "db1";

    tmpHDFSDir = new Path("/tmp/external");
    dbNames = new String[]{dbName};
    roles = new String[]{"admin_role"};
    admin = StaticUserGroup.ADMIN1;

    Connection conn;
    Statement stmt;

    conn = hiveServer2.createConnection("hive", "hive");
    stmt = conn.createStatement();

    stmt.execute("create role admin_role");
    stmt.execute("grant all on server server1 to role admin_role");
    stmt.execute("grant role admin_role to group " + StaticUserGroup.ADMINGROUP);

    conn = hiveServer2.createConnection(StaticUserGroup.ADMIN1, StaticUserGroup.ADMIN1);
    stmt = conn.createStatement();
    try {
      stmt.execute("create database " + dbName);
      stmt.execute("create table test(a string)");
      stmt.execute("create view testView as select * from test");
      stmt.execute("create or replace view testView as select * from test");
      stmt.execute("drop view testView");
    } catch(Exception s) {
      throw s;
    }

    stmt.close();
    conn.close();
  }

  //SENTRY-884
  @Test
  public void testAccessToTableDirectory() throws Throwable {
    String dbName= "db1";

    tmpHDFSDir = new Path("/tmp/external");
    dbNames = new String[]{dbName};
    roles = new String[]{"admin_role", "table_role"};
    admin = StaticUserGroup.ADMIN1;

    Connection conn;
    Statement stmt;

    conn = hiveServer2.createConnection("hive", "hive");
    stmt = conn.createStatement();

    stmt.execute("create role admin_role");
    stmt.execute("grant all on server server1 to role admin_role");
    stmt.execute("grant role admin_role to group " + StaticUserGroup.ADMINGROUP);

    conn = hiveServer2.createConnection(StaticUserGroup.ADMIN1, StaticUserGroup.ADMIN1);
    stmt = conn.createStatement();
    stmt.execute("create database " + dbName);
    stmt.execute("use " + dbName);
    stmt.execute("create table tb1(a string)");

    stmt.execute("create role table_role");
    stmt.execute("grant all on table tb1 to role table_role");
    stmt.execute("grant role table_role to group " + StaticUserGroup.USERGROUP1);
    Thread.sleep(CACHE_REFRESH);//Wait till sentry cache is updated in Namenode
    //Verify user1 is able to access table directory
    verifyAccessToPath(StaticUserGroup.USER1_1, StaticUserGroup.USERGROUP1, "/user/hive/warehouse/db1.db/tb1", true);

    stmt.close();
    conn.close();
  }

  private void verifyAccessToPath(String user, String group, String path, boolean hasPermission) throws Exception{
    Path p = new Path(path);
    UserGroupInformation hadoopUser =
        UserGroupInformation.createUserForTesting(user, new String[] {group});
    FileSystem fs = DFSTestUtil.getFileSystemAs(hadoopUser, hadoopConf);
    try {
      fs.listFiles(p, true);
      if(!hasPermission) {
        Assert.assertFalse("Expected listing files to fail", false);
      }
    } catch (Exception e) {
      if(hasPermission) {
        throw e;
      }
    }
  }

  private void verifyQuery(Statement stmt, String table, int n) throws Throwable {
    verifyQuery(stmt, table, n, NUM_RETRIES);
  }

  private void verifyQuery(Statement stmt, String table, int n, int retry) throws Throwable {
    ResultSet rs = null;
    try {
      rs = stmt.executeQuery("select * from " + table);
      int numRows = 0;
      while (rs.next()) { numRows++; }
      Assert.assertEquals(n, numRows);
    } catch (Throwable th) {
      if (retry > 0) {
        Thread.sleep(RETRY_WAIT);
        verifyQuery(stmt, table, n, retry - 1);
      } else {
        throw th;
      }
    }
  }

  private void loadData(Statement stmt) throws IOException, SQLException {
    FSDataOutputStream f1 = miniDFS.getFileSystem().create(new Path("/tmp/f1.txt"));
    f1.writeChars("m1d1_t1\n");
    f1.writeChars("m1d1_t2\n");
    f1.writeChars("m1d1_t3\n");
    f1.flush();
    f1.close();
    stmt.execute("load data inpath \'/tmp/f1.txt\' overwrite into table p1 partition (month=1, day=1)");
    FSDataOutputStream f2 = miniDFS.getFileSystem().create(new Path("/tmp/f2.txt"));
    f2.writeChars("m2d2_t4\n");
    f2.writeChars("m2d2_t5\n");
    f2.writeChars("m2d2_t6\n");
    f2.flush();
    f2.close();
    stmt.execute("load data inpath \'/tmp/f2.txt\' overwrite into table p1 partition (month=2, day=2)");
    ResultSet rs = stmt.executeQuery("select * from p1");
    List<String> vals = new ArrayList<String>();
    while (rs.next()) {
      vals.add(rs.getString(1));
    }
    Assert.assertEquals(6, vals.size());
    rs.close();
  }

  private void loadDataTwoCols(Statement stmt) throws IOException, SQLException {
    FSDataOutputStream f1 = miniDFS.getFileSystem().create(new Path("/tmp/f2.txt"));
    f1.writeChars("m1d1_t1, m1d1_t2\n");
    f1.writeChars("m1d1_t2, m1d1_t2\n");
    f1.writeChars("m1d1_t3, m1d1_t2\n");
    f1.flush();
    f1.close();
    stmt.execute("load data inpath \'/tmp/f2.txt\' overwrite into table p1 partition (month=1, day=1)");
    ResultSet rs = stmt.executeQuery("select * from p1");
    List<String> vals = new ArrayList<String>();
    while (rs.next()) {
      vals.add(rs.getString(1));
    }
    Assert.assertEquals(3, vals.size());
    rs.close();
  }

  private void writeToPath(String path, int numRows, String user, String group) throws IOException {
    Path p = new Path(path);
    miniDFS.getFileSystem().mkdirs(p);
    miniDFS.getFileSystem().setOwner(p, user, group);
//    miniDFS.getFileSystem().setPermission(p, FsPermission.valueOf("-rwxrwx---"));
    FSDataOutputStream f1 = miniDFS.getFileSystem().create(new Path(path + "/stuff.txt"));
    for (int i = 0; i < numRows; i++) {
      f1.writeChars("random" + i + "\n");
    }
    f1.flush();
    f1.close();
    miniDFS.getFileSystem().setOwner(new Path(path + "/stuff.txt"), "asuresh", "supergroup");
    miniDFS.getFileSystem().setPermission(new Path(path + "/stuff.txt"), FsPermission.valueOf("-rwxrwx---"));
  }

  private void verifyHDFSandMR(Statement stmt) throws Throwable {
    // hbase user should not be allowed to read...
    UserGroupInformation hbaseUgi = UserGroupInformation.createUserForTesting("hbase", new String[] {"hbase"});
    hbaseUgi.doAs(new PrivilegedExceptionAction<Void>() {
      @Override
      public Void run() throws Exception {
        try {
          miniDFS.getFileSystem().open(new Path("/user/hive/warehouse/p1/month=1/day=1/f1.txt"));
          Assert.fail("Should not be allowed !!");
        } catch (Exception e) {
          Assert.assertEquals("Wrong Error : " + e.getMessage(), true, e.getMessage().contains("Permission denied: user=hbase"));
        }
        return null;
      }
    });

    // WordCount should fail..
    // runWordCount(new JobConf(miniMR.getConfig()), "/user/hive/warehouse/p1/month=1/day=1", "/tmp/wc_out");

    stmt.execute("grant select on table p1 to role p1_admin");

    verifyOnAllSubDirs("/user/hive/warehouse/p1", FsAction.READ_EXECUTE, "hbase", true);
    // hbase user should now be allowed to read...
    hbaseUgi.doAs(new PrivilegedExceptionAction<Void>() {
      @Override
      public Void run() throws Exception {
        Path p = new Path("/user/hive/warehouse/p1/month=2/day=2/f2.txt");
        BufferedReader in = new BufferedReader(new InputStreamReader(miniDFS.getFileSystem().open(p)));
        String line = null;
        List<String> lines = new ArrayList<String>();
        do {
          line = in.readLine();
          if (line != null) lines.add(line);
        } while (line != null);
        Assert.assertEquals(3, lines.size());
        in.close();
        return null;
      }
    });

  }

  private void verifyOnAllSubDirs(String path, FsAction fsAction, String group, boolean groupShouldExist) throws Throwable {
    verifyOnAllSubDirs(path, fsAction, group, groupShouldExist, true);
  }

  private void verifyOnPath(String path, FsAction fsAction, String group, boolean groupShouldExist) throws Throwable {
    verifyOnAllSubDirs(path, fsAction, group, groupShouldExist, false);
  }

  private void verifyOnAllSubDirs(String path, FsAction fsAction, String group, boolean groupShouldExist, boolean recurse) throws Throwable {
    verifyOnAllSubDirs(new Path(path), fsAction, group, groupShouldExist, recurse, NUM_RETRIES);
  }

  private void verifyOnAllSubDirs(Path p, FsAction fsAction, String group, boolean groupShouldExist, boolean recurse, int retry) throws Throwable {
    FileStatus fStatus = null;
    try {
      fStatus = miniDFS.getFileSystem().getFileStatus(p);
      if (groupShouldExist) {
        Assert.assertEquals("Error at verifying Path action : " + p + " ;", fsAction, getAcls(p).get(group));
      } else {
        Assert.assertFalse("Error at verifying Path : " + p + " ," +
                " group : " + group + " ;", getAcls(p).containsKey(group));
      }
    } catch (Throwable th) {
      if (retry > 0) {
        Thread.sleep(RETRY_WAIT);
        verifyOnAllSubDirs(p, fsAction, group, groupShouldExist, recurse, retry - 1);
      } else {
        throw th;
      }
    }
    if (recurse) {
      if (fStatus.isDirectory()) {
        FileStatus[] children = miniDFS.getFileSystem().listStatus(p);
        for (FileStatus fs : children) {
          verifyOnAllSubDirs(fs.getPath(), fsAction, group, groupShouldExist, recurse, NUM_RETRIES);
        }
      }
    }
  }

  private Map<String, FsAction> getAcls(Path path) throws Exception {
    AclStatus aclStatus = miniDFS.getFileSystem().getAclStatus(path);
    Map<String, FsAction> acls = new HashMap<String, FsAction>();
    for (AclEntry ent : aclStatus.getEntries()) {
      if (ent.getType().equals(AclEntryType.GROUP)) {
        acls.put(ent.getName(), ent.getPermission());
      }
    }
    return acls;
  }

  private void runWordCount(JobConf job, String inPath, String outPath) throws Exception {
    Path in = new Path(inPath);
    Path out = new Path(outPath);
    miniDFS.getFileSystem().delete(out, true);
    job.setJobName("TestWC");
    JobClient jobClient = new JobClient(job);
    RunningJob submittedJob = null;
    FileInputFormat.setInputPaths(job, in);
    FileOutputFormat.setOutputPath(job, out);
    job.set("mapreduce.output.textoutputformat.separator", " ");
    job.setInputFormat(TextInputFormat.class);
    job.setMapOutputKeyClass(Text.class);
    job.setMapOutputValueClass(LongWritable.class);
    job.setOutputKeyClass(Text.class);
    job.setOutputValueClass(LongWritable.class);
    job.setMapperClass(WordCountMapper.class);
    job.setReducerClass(SumReducer.class);
    job.setOutputFormat(TextOutputFormat.class);
    job.setNumReduceTasks(1);
    job.setInt("mapreduce.map.maxattempts", 1);
    job.setInt("mapreduce.reduce.maxattempts", 1);

    submittedJob = jobClient.submitJob(job);
    if (!jobClient.monitorAndPrintJob(job, submittedJob)) {
      throw new IOException("job Failed !!");
    }

  }

}