text:
<?php

class SpecParserException extends Exception {
}

class GadgetSpecParser {

  public function parse($xml, $context) {
    if (empty($xml)) {
      throw new SpecParserException("Empty XML document");
    }
    // make sure we can generate a detailed error report
    libxml_use_internal_errors(true);
    if (($doc = simplexml_load_string($xml, 'SimpleXMLElement', LIBXML_NOCDATA)) == false) {
      $errors = @libxml_get_errors();
      $xmlErrors = '';
      foreach ($errors as $error) {
        $xmlErrors .= $this->displayXmlError($error);
      }
      @libxml_clear_errors();
      throw new SpecParserException("Invalid XML Document
\n" . $xmlErrors);
    }
    if (count($doc->ModulePrefs) != 1) {
      throw new SpecParserException("Missing or duplicated <ModulePrefs>");
    }
    $gadget = new Gadget($context->getGadgetId(), $context);
    // record Checksum to trace xml version
    $gadget->setChecksum($xml);
    // process ModulePref attributes
    $this->processModulePrefs($gadget, $doc->ModulePrefs);
    if (isset($doc->ModulePrefs->OAuth)) {
      // process OAuthPref attributes
      $this->processOAuthSpec($gadget, $doc->ModulePrefs->OAuth, $context);
    }
    // process UserPrefs, if any
    foreach ($doc->UserPref as $pref) {
      $this->processUserPref($gadget, $pref);
    }
    // Assume gadget v1
    $explicit_profile = false;
    foreach ($doc->Content as $content) {
      $attributes = $content->attributes();
      if (empty($attributes['type'])) {
        throw new SpecParserException("No content type specified!");
      }
      $view = isset($attributes['view']) ? trim($attributes['view']) : '';
      // Note if we find a profile explicity defined
      if (strpos($view, "profile") !== false) {
        $explicit_profile = true;
      }
    }
    foreach ($doc->Content as $content) {
      $attributes = $content->attributes();
      if (empty($attributes['type'])) {
        throw new SpecParserException("No content type specified!");
      }
      $view = isset($attributes['view']) ? trim($attributes['view']) : '';
      // If view isnt defined and we didnt find a profile explicity defined
      if ($view == '') {
        if (! $explicit_profile) {
          $this->processContent($gadget, $content, array(0 => DEFAULT_VIEW));
        }
        // If view isnt defined and a profile was found, this will catch it
      } else {
        $views = explode(',', str_replace(' ', '', $view));
        $this->processContent($gadget, $content, $views);
      }
    }
    foreach ($doc->ModulePrefs->Preload as $feature) {
      $gadget->preloads[] = new Preload($feature);
    }
    foreach ($doc->ModulePrefs->Require as $feature) {
      $this->processFeature($gadget, $feature, true);
    }
    foreach ($doc->ModulePrefs->Optional as $feature) {
      $this->processFeature($gadget, $feature, false);
    }
    foreach ($doc->ModulePrefs->Icon as $icon) {
      $this->processIcon($gadget, $icon);
    }
    return $gadget;
  }

  private function processModulePrefs(&$gadget, $ModulePrefs) {
    $attributes = $ModulePrefs->attributes();
    if (empty($attributes['title'])) {
      throw new SpecParserException("Missing or empty \"title\" attribute.");
    }
    // Get ModulePrefs base and extended attributes
    // See http://code.google.com/apis/gadgets/docs/gadgets-extended-xsd.html
    // (trim is used here since it not only cleans up the text, but also auto-casts the SimpleXMLElement to a string)
    $gadget->title = trim($attributes['title']);
    $gadget->author = isset($attributes['author']) ? trim($attributes['author']) : '';
    $gadget->authorEmail = isset($attributes['author_email']) ? trim($attributes['author_email']) : '';
    $gadget->description = isset($attributes['description']) ? trim($attributes['description']) : '';
    $gadget->directoryTitle = isset($attributes['directory_title']) ? trim($attributes['directory_title']) : '';
    $gadget->screenshot = isset($attributes['screenshot']) ? trim($attributes['screenshot']) : '';
    $gadget->thumbnail = isset($attributes['thumbnail']) ? trim($attributes['thumbnail']) : '';
    $gadget->titleUrl = isset($attributes['title_url']) ? trim($attributes['title_url']) : '';
    // Extended spec fields
    $gadget->authorAffiliation = isset($attributes['author_affiliation']) ? trim($attributes['author_affiliation']) : '';
    $gadget->authorLocation = isset($attributes['author_location']) ? trim($attributes['author_location']) : '';
    $gadget->authorPhoto = isset($attributes['author_photo']) ? trim($attributes['author_photo']) : '';
    $gadget->authorAboutMe = isset($attributes['author_aboutme']) ? trim($attributes['author_aboutme']) : '';
    $gadget->authorQuote = isset($attributes['author_quote']) ? trim($attributes['author_quote']) : '';
    $gadget->authorLink = isset($attributes['author_link']) ? trim($attributes['author_link']) : '';
    $gadget->showStats = isset($attributes['show_stats']) ? trim($attributes['show_stats']) : '';
    $gadget->showInDirectory = isset($attributes['show_in_directory']) ? trim($attributes['show_in_directory']) : '';
    $gadget->string = isset($attributes['string']) ? trim($attributes['string']) : '';
    $gadget->width = isset($attributes['width']) ? trim($attributes['width']) : '';
    $gadget->height = isset($attributes['height']) ? trim($attributes['height']) : '';
    $gadget->category = isset($attributes['category']) ? trim($attributes['category']) : '';
    $gadget->category2 = isset($attributes['category2']) ? trim($attributes['category2']) : '';
    $gadget->singleton = isset($attributes['singleton']) ? trim($attributes['singleton']) : '';
    $gadget->renderInline = isset($attributes['render_inline']) ? trim($attributes['render_inline']) : '';
    $gadget->scaling = isset($attributes['scaling']) ? trim($attributes['scaling']) : '';
    $gadget->scrolling = isset($attributes['scrolling']) ? trim($attributes['scrolling']) : '';
    foreach ($ModulePrefs->Link as $link) {
      $gadget->links[] = $this->processLink($link);
    }
    foreach ($ModulePrefs->Locale as $locale) {
      $gadget->localeSpecs[] = $this->processLocale($locale);
    }
  }

  private function processLink($link) {
    $attributes = $link->attributes();
    $rel = isset($attributes['rel']) ? trim($attributes['rel']) : '';
    $href = isset($attributes['href']) ? trim($attributes['href']) : '';
    $method = isset($attributes['method']) ? trim($attributes['method']) : 'GET';
    $link = new LinkSpec($rel, $href, $method);
    return $link;
  }

  private function processLocale($locale) {
    $attributes = $locale->attributes();
    $messageAttr = isset($attributes['messages']) ? trim($attributes['messages']) : '';
    $languageAttr = isset($attributes['lang']) ? trim($attributes['lang']) : 'all';
    $countryAttr = isset($attributes['country']) ? trim($attributes['country']) : 'all';
    $rtlAttr = isset($attributes['language_direction']) ? trim($attributes['language_direction']) : '';
    $rightToLeft = $rtlAttr == 'rtl';
    $localeMessageBundles = array();
    if ($messageAttr == '') {
      $parser = new MessageBundleParser();
      $localeMessageBundles = $parser->getMessages($locale);
    }
    $locale = new LocaleSpec();
    $locale->rightToLeft = $rightToLeft;
    $locale->url = $messageAttr;
    $locale->localeMessageBundles = $localeMessageBundles;
    $locale->locale = new Locale($languageAttr, $countryAttr);
    return $locale;
  }

  private function processUserPref(&$gadget, $pref) {
    $attributes = $pref->attributes();
    $preference = new UserPref();
    if (empty($attributes['name'])) {
      throw new SpecParserException("All UserPrefs must have name attributes.");
    }
    $preference->name = trim($attributes['name']);
    $preference->displayName = isset($attributes['display_name']) ? $gadget->getSubstitutions()->substitute(trim($attributes['display_name'])) : '';
    // if its set -and- in our valid 'enum' of types, use it, otherwise assume STRING, to try and emulate java's enum behavior
    $preference->required = isset($attributes['required']) ? $gadget->getSubstitutions()->substitute(trim($attributes['required'])) : 'false';
    $preference->dataType = isset($attributes['datatype']) && in_array(strtoupper($attributes['datatype']), $preference->DataTypes) ? strtoupper($attributes['datatype']) : 'STRING';
    $preference->defaultValue = isset($attributes['default_value']) ? $gadget->getSubstitutions()->substitute(trim($attributes['default_value'])) : '';
    if (isset($pref->EnumValue)) {
      foreach ($pref->EnumValue as $enum) {
        $attr = $enum->attributes();
        $valueText = trim($attr['value']);
        $displayText = ! empty($attr['display_value']) ? $gadget->getSubstitutions()->substitute(trim($attr['display_value'])) : $valueText;
        $preference->enumValues[$valueText] = $displayText;
      }
    }
    $gadget->userPrefs[] = $preference;
  }

  private function processContent(&$gadget, $content, $views) {
    $html = (string)$content; // no trim here since empty lines can have structural meaning, so typecast to string instead
    foreach ($views as $view) {
      $viewSpec = new ViewSpec($view, $content);
      if (! isset($gadget->views[$view])) {
        $viewSpec->content = $html;
        $gadget->views[$view] = $viewSpec;
      } else {
        if ($gadget->views[$view]->getName() == $viewSpec->getName() && $viewSpec->getType() != $gadget->views[$view]->getType()) {
          throw new SpecParserException("You may not mix content " . " types in the same view.");
        }
        $gadget->views[$view]->addContent($html);
      }
    }
  }

  private function processFeature(&$gadget, $feature, $required) {
    $featureSpec = new FeatureSpec();
    $attributes = $feature->attributes();
    if (empty($attributes['feature'])) {
      throw new SpecParserException("Feature not specified in <" . ($required ? "Required" : "Optional") . "> tag");
    }
    $featureSpec->name = trim($attributes['feature']);
    $featureSpec->optional = ! $required;
    foreach ($feature->Param as $param) {
      $attr = $param->attributes();
      if (empty($attr['name'])) {
        throw new SpecParserException("Missing name attribute in <Param>.");
      }
      $name = trim($attr['name']);
      $value = trim($param);
      $featureSpec->params[$name] = $value;
    }
    $gadget->requires[$featureSpec->name] = $featureSpec;
  }

  private function processIcon(Gadget &$gadget, $icon) {
    $attributes = $icon->attributes();
    $iconSpec = new Icon();
    $iconSpec->content = (string)(trim($icon));
    $iconSpec->mode = isset($attributes['mode']) ? trim($attributes['mode']) : '';
    $iconSpec->type = isset($attributes['type']) ? trim($attributes['type']) : '';
    $gadget->icons[] = $iconSpec;
  }

  private function processOAuthSpec(Gadget &$gadget, $OAuthSpec) {
    $oauthSpec = new OAuthSpec($OAuthSpec->Service);
    $gadget->setOAuthSpec($oauthSpec);
  }

  private function displayXmlError($error) {
    $return = '';
    switch ($error->level) {
      case LIBXML_ERR_WARNING:
        $return .= "Warning [{$error->code}]: ";
        break;
      case LIBXML_ERR_ERROR:
        $return .= "Error [{$error->code}]: ";
        break;
      case LIBXML_ERR_FATAL:
        $return .= "Fatal Error [{$error->code}]: ";
        break;
    }
    $return .= trim($error->message) . ", Line: {$error->line}" . " Column: {$error->column}";
    if ($error->file) {
      $return .= "File: {$error->file}";
    }
    return "$return\n";
  }
}


text: overhaul my blog!

redpajama_set_name: RedPajamaGithub


text:
001
018
019    package org.apache.hadoop.hdfs.server.namenode;
020
021    import java.io.IOException;
022    import org.apache.hadoop.classification.InterfaceAudience;
023    import org.apache.hadoop.classification.InterfaceStability;
024
025    import org.apache.commons.logging.Log;
026    import org.apache.commons.logging.LogFactory;
027
028    /** Context data for an ongoing NameNode metadata recovery process. */
029    @InterfaceAudience.Private
030    @InterfaceStability.Evolving
031    public final class MetaRecoveryContext  {
032      public static final Log LOG = LogFactory.getLog(MetaRecoveryContext.class.getName());
033      public final static int FORCE_NONE = 0;
034      public final static int FORCE_FIRST_CHOICE = 1;
035      public final static int FORCE_ALL = 2;
036      private int force;
037
038      /** Exception thrown when the user has requested processing to stop. */
039      static public class RequestStopException extends IOException {
040        private static final long serialVersionUID = 1L;
041        public RequestStopException(String msg) {
042          super(msg);
043        }
044      }
045
046      public MetaRecoveryContext(int force) {
047        this.force = force;
048      }
049
050      /**
051       * Display a prompt to the user and get his or her choice.
052       *
053       * @param prompt      The prompt to display
054       * @param default     First choice (will be taken if autoChooseDefault is
055       *                    true)
056       * @param choices     Other choies
057       *
058       * @return            The choice that was taken
059       * @throws IOException
060       */
061      public String ask(String prompt, String firstChoice, String... choices)
062          throws IOException {
063        while (true) {
064          LOG.info(prompt);
065          if (force > FORCE_NONE) {
066            LOG.info("automatically choosing " + firstChoice);
067            return firstChoice;
068          }
069          StringBuilder responseBuilder = new StringBuilder();
070          while (true) {
071            int c = System.in.read();
072            if (c == -1 || c == '\r' || c == '\n') {
073              break;
074            }
075            responseBuilder.append((char)c);
076          }
077          String response = responseBuilder.toString();
078          if (response.equalsIgnoreCase(firstChoice))
079            return firstChoice;
080          for (String c : choices) {
081            if (response.equalsIgnoreCase(c)) {
082              return c;
083            }
084          }
085          LOG.error("I'm sorry, I cannot understand your response.\n");
086        }
087      }
088
089      public static void editLogLoaderPrompt(String prompt,
090            MetaRecoveryContext recovery, String contStr)
091            throws IOException, RequestStopException
092      {
093        if (recovery == null) {
094          throw new IOException(prompt);
095        }
096        LOG.error(prompt);
097        String answer = recovery.ask("\nEnter 'c' to continue, " + contStr + "\n" +
098          "Enter 's' to stop reading the edit log here, abandoning any later " +
099            "edits\n" +
100          "Enter 'q' to quit without saving\n" +
101          "Enter 'a' to always select the first choice in the future " +
102          "without prompting. " +
103          "(c/s/q/a)\n", "c", "s", "q", "a");
104        if (answer.equals("c")) {
105          LOG.info("Continuing");
106          return;
107        } else if (answer.equals("s")) {
108          throw new RequestStopException("user requested stop");
109        } else if (answer.equals("q")) {
110          recovery.quit();
111        } else {
112          recovery.setForce(FORCE_FIRST_CHOICE);
113          return;
114        }
115      }
116
117      /** Log a message and quit */
118      public void quit() {
119        LOG.error("Exiting on user request.");
120        System.exit(0);
121      }
122
123      public int getForce() {
124        return this.force;
125      }
126
127      public void setForce(int force) {
128        this.force = force;
129      }
130    }


text:
id: 587d7dad367417b2b2512b75
title: Create a Method on an Object
challengeType: 1
isHidden: false
forumTopicId: 301318
---

## Description

Objects can have a special type of property, called a <dfn>method</dfn>.
Methods are properties that are functions. This adds different behavior to an object. Here is the duck example with a method:

```js
let duck = {
  name: "Aflac",
  numLegs: 2,
  sayName: function() {return "The name of this duck is " + duck.name + ".";}
};
duck.sayName();
// Returns "The name of this duck is Aflac."
```

The example adds the sayName method, which is a function that returns a sentence giving the name of the duck.
Notice that the method accessed the name property in the return statement using duck.name. The next challenge will cover another way to do this.

## Instructions

Using the dog object, give it a method called sayLegs. The method should return the sentence "This dog has 4 legs."

## Tests

```yml
tests:
  - text: <code>dog.sayLegs()</code> should be a function.
    testString: assert(typeof(dog.sayLegs) === 'function');
  - text: <code>dog.sayLegs()</code> should return the given string - note that punctuation and spacing matter.
    testString: assert(dog.sayLegs() === 'This dog has 4 legs.');

```

## Challenge Seed

```js
let dog = {
  name: "Spot",
  numLegs: 4,

};

dog.sayLegs();
```

## Solution

```js
let dog = {
  name: "Spot",
  numLegs: 4,
  sayLegs () {
    return 'This dog has ' + this.numLegs + ' legs.';
  }
};

dog.sayLegs();
```


text:
var ScriptInspect = React.createClass({
	onEditScript:function(e){
		var name=this.props.name
		// console.log(this.props.name)
		fetch('/script'+name, {
	  		method: 'GET',
			headers: {
				'Accept': 'application/json',
				'Content-Type': 'application/json'
			}
		}).then(function(response) {
			// console.log(response.text())
		    return response.text()
		}).then(function(data){
			// console.log(data)
			store.dispatch({type:'SCRIPT_EDIT',name:name,data:data})
		}).catch(function(ex) {
		    console.log('parsing failed', ex)
		})
	},
	onDeleteScript:function(){
		if(confirm("Are you sure you wan't to delete the script "+this.props.name+"? This can NOT be undone!!!")){
			console.log('delete')
			var name=this.props.name
			// console.log(this.props.name)
			fetch('/script'+name, {
		  		method: 'DELETE',
				headers: {
					'Accept': 'application/json',
					'Content-Type': 'application/json'
				}
			}).then(function(response) {
				// console.log(response.text())
			    return response.text()
			}).then(function(data){
				// console.log(data)
				// store.dispatch({type:'SCRIPT_EDIT',name:name,data:data})
				updateSources();
			}).catch(function(ex) {
			    console.log('parsing failed', ex)
			})
		}
	},
	render:function(){
		var fields=[]
		fields.push(h('input.form_button',{'type':'button','value':'Edit','onClick':this.onEditScript}))
		fields.push(h('div.form_spacer'))
		fields.push(h('input.form_button',{'type':'button','value':'Delete script','onClick':this.onDeleteScript}))
		return h('div.inspector',fields)
	}
})

function updateScriptRunner(){
	var state=store.getState()

	var source=state.script_editor.sample_source
	if(source==null){
		// TODO: such a hack - set all of this up when "new script" is clicked
		source=state.streams[0].topic
	}
	// console.log('source')
	// console.log(source)
	if(source!=null){
		(function(){
			try{
				var stroom={
					openStream:function(name){
						return {
							get:function(index,endIndex){
								return []
							},
							get:function(index){
								return ""
							},
							getLast:function(){
								return ""
							},
							getCount:function(){
								return 0
							},
							append:function(data){
								console.log(data)
								return 0
							},
							append:function(data,hint){
								console.log(data)
								return 0
							},
							truncate:function(index){

							}
						}
					}
				}
				eval(state.script_editor.data)
				store.dispatch({type:'SET',path:['script_editor','eval_success'],value:null})
				var count=getStream(state.streams,source).count
				if(count>state.script_editor.sample_count){
					var loc=Math.floor(Math.random()*(count-parseInt(state.script_editor.sample_count)))

					// console.log(loc)
					// Get count

					fetch('/stream/'+source+'/'+loc+'-'+(loc+parseInt(state.script_editor.sample_count-1)),{
				  		method: 'GET',
						headers: {
							'Accept': 'application/json',
							'Content-Type': 'application/json'
						}
					}).then(function(response) {
					    return response.json()
					}).then(function(json) {
						try{
							// console.log('should be running')
							var ftype=state.script_editor.sample_function
							var data=[]
							var result=null
							// console.log('count: '+json.length)
							// console.log('selected:'+state.script_editor.sample_count)
							for(var i=0;i<json.length;i++){
								var doc=JSON.parse(JSON.stringify(json[i]))
								if(ftype=='map'){
									result=map(json[i])
								}else if(ftype=='reduce'){
									result=reduce(result,json[i])
								}
								data.push({
									'doc':doc,'result':JSON.parse(JSON.stringify(result))
								})
							}
							store.dispatch({type:'SET',path:['script_editor','sample_data'],value:data})
						}catch(inner_ex){
							// console.log(inner_ex)
							store.dispatch({type:'SCRIPT_EVAL',success:false,error:inner_ex})
						}
					})
				}else{
					store.dispatch({type:'SCRIPT_EVAL',success:false,error:'The input stream needs to have at least '+state.script_editor.sample_count+' documents.'})
				}
			}catch(ex){
				// console.log(ex)
				store.dispatch({type:'SCRIPT_EVAL',success:false,error:ex})
				return
			}
		})();
	}
}

function getStream(streams,topic){
	for(var i=0;i<streams.length;i++){
		var stream=streams[i]
		if(stream.topic==topic)return stream
	}
	return null
}

var SampleResults=React.createClass({
	render:function(){
		if(this.props.sample_data.length==0){
			return h('div','')
		}else{
			var rendered=[]
			rendered.push(h('div.split_view',[h('div.split_header','Input'),h('div.split_header','Output')]))
			for(var i=0;i<this.props.sample_data.length;i++){
				var data=this.props.sample_data[i]
				rendered.push(h('div.split_view',[h('div.split',[h(JSONViewer,{'data':data.doc})]),h('div.split',[h(JSONViewer,{'data':data.result})])]))
			}
			rendered.push(h('div.split_view',[h('div'),h('div')]))
			return h('div',rendered)
		}
	}
})/*

var SampleRunner = React.createClass({
	runTest:function(e){
		console.log('click click')
		// store.dispatch({type:'SET',path:['script_editor','sample_run'],value:true})
		// updateScriptRunner()
	},
	setIn:function(e){
		store.dispatch({type:'SET',path:['script_editor','sample_source'],value:e.target.value})
	},
	render:function(){
		var elements=[]

		// elements.push(h('div.browse_header','Edit Script'))

		var options=[]
		for(var i=0;i<this.props.streams.length;i++){
			options.push(h('option',{value:this.props.streams[i].topic},this.props.streams[i].topic))
		}

		var footer=[]
		footer.push(h('label.form_label','Input Stream'))
		footer.push(h('select.form_select',{name:'in_stream_select',onChange:this.setIn},options))
		footer.push(h('label.form_label','Function'))
		options=[]
		options.push(h('option',{value:'map'},'Map'))
		options.push(h('option',{value:'reduce'},'Reduce'))
		footer.push(h('select.form_select',{name:'function_select',onChange:this.setFunction},options))
		footer.push(h('input.form_button',{'type':'button','value':'Run','onClick':this.runTest}))

		elements.push(h('div.browse_footer',footer))

		elements.push(h(SampleResults,this.props.script_editor))
		return h('div',elements)
	}
})*/

var ScriptEditor =React.createClass({
	onSave:function(e){
		var filename=this.props.script_editor.name
		if(!filename.startsWith('/')){
			filename='/'+filename;
		}
		fetch('/script'+filename, {
		  		method: 'POST',
				headers: {
					'Accept': 'application/json',
					'Content-Type': 'application/json'
				},
				body:this.props.script_editor.data
			}).then(function(response) {
			    return response.json()
			}).then(function(json) {
	            console.log(json)
				// store.dispatch({type:'SERVER_STREAMS',data:json})
				// store.dispatch({type:'SET',path:['currentStream'],value:null})
				// updateSources();
			}).catch(function(ex) {
			    console.log('parsing failed', ex)
			})
	},
	onCancel:function(e){
		store.dispatch({type:'SET',path:['script_editor','show'],value:false})
	},
	saveFilename:function(e){
		store.dispatch({type:'SET',path:['script_editor','name'],value:e.target.value})
	},
	saveScript:function(e){
		store.dispatch({type:'SET',path:['script_editor','data'],value:e.target.value})
		// console.log(e.target.value)
	},
	onEval:function(e){
		try{
			(eval(this.props.script_editor.data))
			store.dispatch({type:'SCRIPT_EVAL',success:true,error:null})
		}catch(ex){
			console.log(ex)
			store.dispatch({type:'SCRIPT_EVAL',success:false,error:ex})
		}
	},
	onKeyDown:function(e){
		if(e.keyCode === 9){
	        var i1 = e.target.selectionStart
	        var i2 = e.target.selectionEnd
	        var target = e.target
	        var value = target.value
	        target.value = value.substring(0, i1) + "\t"+ value.substring(i2)
	        e.target.selectionStart = e.target.selectionEnd = i1 + 1
	        e.preventDefault()
	    }
	},
	runTest:function(e){
		store.dispatch({type:'SET',path:['script_editor','sample_run'],value:true})
		updateScriptRunner()
	},
	setIn:function(e){
		store.dispatch({type:'SET',path:['script_editor','sample_source'],value:e.target.value})
	},
	setFType:function(e){
		store.dispatch({type:'SET',path:['script_editor','sample_function'],value:e.target.value})
	},
	setSampleCount:function(e){
		store.dispatch({type:'SET',path:['script_editor','sample_count'],value:e.target.value})
	},
	render:function(){
		var elements=[]
		elements.push(h('div.browse_header','Edit Script'))
		elements.push(h('div.browse_page',[
			h('input',{'defaultValue':this.props.script_editor.name,'name':'name',onChange:this.saveFilename}),
			h('textarea.editor',{'defaultValue':this.props.script_editor.data,'name':'script_content',onChange:this.saveScript,onKeyDown:this.onKeyDown})
			]))
		if(this.props.script_editor.eval_success!=null){
			if(this.props.script_editor.eval_success){
				elements.push(h('div.eval_success','The script was evaluated without error!'))
			}else{
				console.log()
				elements.push(h('div.eval_error',[h('div',this.props.script_editor.eval_error.toString())]))
			}
		}
		var footer=[
				h('input.form_button',{'type':'button','value':'Save','onClick':this.onSave}),
				h('input.form_button',{'type':'button','value':'Cancel','onClick':this.onCancel}),
				h('div.form_spacer',''),
				h('input.form_button',{'type':'button','value':'Eval Script','onClick':this.onEval}),
				h('div.form_spacer','')
			]
		footer.push(h('label.form_label','In'))
		var options=[]
		for(var i=0;i<this.props.streams.length;i++){
			options.push(h('option',{value:this.props.streams[i].topic},this.props.streams[i].topic))
		}
		footer.push(h('select.form_select',{name:'in_stream_select',onChange:this.setIn},options))
		footer.push(h('label.form_label','Samples'))
		options=[]
		options.push(h('option',{value:'5'},'5'))
		options.push(h('option',{value:'20'},'20'))
		options.push(h('option',{value:'100'},'100'))
		footer.push(h('select.form_select',{name:'sample_select',onChange:this.setSampleCount},options))
		footer.push(h('label.form_label','Function'))
		options=[]
		options.push(h('option',{value:'map'},'Map'))
		options.push(h('option',{value:'reduce'},'Reduce'))
		footer.push(h('select.form_select',{name:'function_select',onChange:this.setFType},options))
		footer.push(h('input.form_button',{'type':'button','value':'Run Test','onClick':this.runTest}))

		elements.push(h('div.browse_footer',footer))
		elements.push(h(SampleResults,this.props.script_editor))
		// elements.push(h('div.browse_footer',))
		return h('div',elements)
	}
})

var ScriptPage = React.createClass({
	onCreateScript:function(){
		store.dispatch({type:'SCRIPT_CREATE'})
	},
	render: function(){
		var currentScript=this.props.currentScript
		var elements=[]
		// Add header
		// elements.push(h('p','So many streams...'))
		// console.log('render '+this.props.currentStream)
		if(this.props.script_editor.show){
			elements.push(h(ScriptEditor,this.props))
			if(this.props.show_sample){
				elements.push(h(SampleRunner,this.props))
			}
		}else{
			elements.push(h('div',[h('input.form_button',{'type':'button','value':'New script','onClick':this.onCreateScript})]))
			elements.push(h(TableView,{id:'scripts',data:this.props.scripts,columns:[
				{key:'name',width:'flex'},
				{key:'size',width:'medium'}
			],'selected':this.props.currentScript
			,'inspectorClass':ScriptInspect
			,'selectRow':function(id){
				if(currentScript==id){
					store.dispatch({type:'SET',path:['currentScript'],value:null})
				}else{
					store.dispatch({type:'SET',path:['currentScript'],value:id})
				}
			}}))
		}
		// Return enclosing div with elements
		return h('div',elements)
	}
})


text:
module Hanuman
  class Rule < ActiveRecord::Base
    has_paper_trail

    # Delegations
    # delegate :update_observation_visibility, to: :condition

    # Constants
    MATCH_TYPES = %w(any all)

    # Relations
    belongs_to :question
    has_many :conditions, dependent: :destroy
    after_commit :update_observation_visibility

    # Validations
    #validates :question_id, presence: true
    validates :match_type, inclusion: { in: MATCH_TYPES }

    amoeba do
      propagate
      exclude_association :conditions
      customize(lambda { |original_rule, new_rule|
        new_rule.duped_rule_id = original_rule.id
      })
    end

    def update_observation_visibility
      unless self.question.blank? || self.question.survey_template.blank?
        self.question.survey_template.surveys.where(observation_visibility_set: true).each do |s|
          s.update_column(:observation_visibility_set, false)
        end
      end
    end
  end
end


text:
package com.google.sitebricks.compiler;

import com.google.common.collect.Maps;
import com.google.common.collect.Sets;
import com.google.inject.Singleton;
import com.google.sitebricks.Evaluator;
import com.google.sitebricks.Visible;
import com.google.sitebricks.conversion.generics.Generics;
import com.google.sitebricks.conversion.generics.ParameterizedTypeImpl;
import org.jetbrains.annotations.Nullable;
import org.mvel2.CompileException;
import org.mvel2.MVEL;
import org.mvel2.ParserContext;
import org.mvel2.compiler.CompiledExpression;
import org.mvel2.compiler.ExpressionCompiler;

import java.beans.IntrospectionException;
import java.beans.Introspector;
import java.beans.PropertyDescriptor;
import java.lang.reflect.Field;
import java.lang.reflect.ParameterizedType;
import java.lang.reflect.Type;
import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;

/**
 * @author Dhanji R. Prasanna (dhanji@gmail com)
 */
@Singleton
public class MvelEvaluatorCompiler implements EvaluatorCompiler {
  private final Class<?> backingType;
  private final Map<String, Type> backingTypes;

  private static final String CLASS = "class";
  private final Set<String> writeableProperties = Sets.newHashSet();
  private final Map<String, Type> egressTypes = Maps.newHashMap();
  private ParserContext cachedParserContext;

  public MvelEvaluatorCompiler(Class<?> backingType) {
    this.backingType = backingType;
    this.backingTypes = null;
  }

  public MvelEvaluatorCompiler(Map<String, Type> backingTypes) {
    this.backingTypes = Collections.unmodifiableMap(backingTypes);
    this.backingType = null;
  }

  //memo field caches compiled expressions
  private final Map<String, CompiledExpression> compiled =
      new HashMap<String, CompiledExpression>();

  public Type resolveEgressType(String expression) throws ExpressionCompileException {

		// try to get the type from the cache
	    Type type = egressTypes.get(expression);
	    if (type != null) {
	    	return type;
	    }

	    CompiledExpression compiled = compileExpression(expression);
		  final Class<?> egressClass = compiled.getKnownEgressType();
	    final Type[] parameters = getParserContext().getLastTypeParameters();

	    if (parameters == null) {
	        // the class is not parameterised (generic)
	    	type = egressClass;
	    }
	    else {
	        // reconstruct the Type from mvel's generics details
	    	type = new ParameterizedTypeImpl(egressClass, parameters, egressClass.getEnclosingClass());
	    }

	    egressTypes.put(expression, type);

	    return type;
  }

  public boolean isWritable(String property) throws ExpressionCompileException {
    // Ensure we have introspected. Relying on sidefx, ugh.
    getParserContext();

    return writeableProperties.contains(property);
  }

  public Evaluator compile(String expression) throws ExpressionCompileException {

    //do *not* inline
    final CompiledExpression compiled = compileExpression(expression);

    return new Evaluator() {
      @Nullable
      public Object evaluate(String expr, Object bean) {
        return MVEL.executeExpression(compiled, bean);
      }

      public void write(String expr, Object bean, Object value) {
        //lets use mvel to store an expression
        MVEL.setProperty(bean, expr, value);
      }

      public Object read(String property, Object contextObject) {
        return MVEL.getProperty(property, contextObject);
      }
    };
  }

  private CompiledExpression compileExpression(String expression)
      throws ExpressionCompileException {
    final CompiledExpression compiledExpression = compiled.get(expression);

    //use cached copy
    if (null != compiledExpression)
      return compiledExpression;

    //otherwise compile expression and cache
    final ExpressionCompiler compiler = new ExpressionCompiler(expression, getParserContext());

    CompiledExpression tempCompiled;
    try {
      tempCompiled = compiler.compile();
    } catch (CompileException ce) {
      throw new ExpressionCompileException(expression, ce.getErrors());
    }

    //store in memo cache
    compiled.put(expression, tempCompiled);

    return tempCompiled;
  }

  private ParserContext getParserContext() throws ExpressionCompileException {
    if (null != cachedParserContext) {
      return cachedParserContext;
    }

    return cachedParserContext = (null != backingType)
        ? singleBackingTypeParserContext() : backingMapParserContext();
  }

  @SuppressWarnings({ "unchecked", "rawtypes" })
private ParserContext backingMapParserContext() {
    ParserContext context = new ParserContext();
    context.setStrongTyping(true);

    context.addInputs((Map) backingTypes);

    return context;
  }

  public List<Token> tokenizeAndCompile(String template) throws ExpressionCompileException {
    return Parsing.tokenize(template, this);
  }

  //generates a parsing context with type information from the backing type's javabean properties
  private ParserContext singleBackingTypeParserContext() throws ExpressionCompileException {
    ParserContext context = new ParserContext();
    context.setStrongTyping(true);
    context.addInput("this", backingType);

    PropertyDescriptor[] propertyDescriptors;
    try {
      propertyDescriptors = Introspector.getBeanInfo(backingType).getPropertyDescriptors();
    } catch (IntrospectionException e) {
      throw new ExpressionCompileException("Could not read class " + backingType);
    }

    // read @Visible annotated fields.
    for (Field field : backingType.getDeclaredFields()) {
      if (field.isAnnotationPresent(Visible.class)) {
        context.addInput(field.getName(), field.getType());

        if (!field.getAnnotation(Visible.class).readOnly()) {
          writeableProperties.add(field.getName());
        }
      }
    }

    // read javabean properties -- these override @Visible fields.
    for (PropertyDescriptor propertyDescriptor : propertyDescriptors) {
      // skip getClass()
      if (CLASS.equals(propertyDescriptor.getName()))
        continue;

      if (null != propertyDescriptor.getWriteMethod()) {
        writeableProperties.add(propertyDescriptor.getName());
      }

      // if this is a collection, determine its type parameter
      if (Collection.class.isAssignableFrom(propertyDescriptor.getPropertyType())) {

        Type propertyType;
        if (propertyDescriptor.getReadMethod() != null) {
          propertyType = propertyDescriptor.getReadMethod().getGenericReturnType();
        }
        else {
          propertyType = propertyDescriptor.getWriteMethod().getGenericParameterTypes()[0];
        }

        ParameterizedType collectionType = (ParameterizedType) Generics
            .getExactSuperType(propertyType, Collection.class);

        Class<?>[] parameterClasses = new Class[1];
        Type parameterType = collectionType.getActualTypeArguments()[0];
        parameterClasses[0] = Generics.erase(parameterType);

        context.addInput(propertyDescriptor.getName(), propertyDescriptor.getPropertyType(), parameterClasses);
      } else {
        context.addInput(propertyDescriptor.getName(), propertyDescriptor.getPropertyType());
      }
    }

    return context;
  }
}


text:
from PIL import Image
import os
import shutil
import json

if not os.path.exists("sorted-characters"):
    os.mkdir("sorted-characters")
else:
    shutil.rmtree('sorted-characters')
    os.mkdir("sorted-characters")

def get_similarity(img_1, img_2):
    pix1 = img_1.convert('L').load()
    pix2 = img_2.convert('L').load()
    match_count = 0
    pix1_black_count = 0
    pix2_black_count = 0

    for y in range(0, img_1.height):
        for x in range(0, img_1.width):
            if pix1[x, y] == pix2[x, y] and pix2[x, y] == 0:
                match_count += 1
            if pix2[x, y] == 0:
                pix2_black_count += 1
            if pix1[x, y] == 0:
                pix1_black_count += 1

    # no of pixels have more than 10% variation
    if float(pix1_black_count - pix2_black_count) / float(pix1_black_count + pix2_black_count) >= 1.5:
        return 0
    return float(match_count)/float(pix2_black_count)

visited_images = []
similarity_dict = {}
count = 0
for image_1_path in os.listdir("cropped-chars"):
    img_1 = Image.open(os.path.join("cropped-chars", image_1_path))
    visited_images.append(image_1_path)
    for image_2_path in os.listdir("cropped-chars"):
        if image_2_path in visited_images:
            continue
        img_2 = Image.open(os.path.join("cropped-chars", image_2_path))
        if get_similarity(img_1, img_2) >= 0.95:
            visited_images.append(image_2_path)
            if image_1_path not in similarity_dict.keys():
                count += 1
                similarity_dict[image_1_path] = [image_2_path]
                os.mkdir(os.path.join("sorted-characters", str(count)))
                img_1.save(os.path.join(
                    "sorted-characters", str(count), image_1_path))
                img_2.save(os.path.join(
                    "sorted-characters", str(count), image_2_path))
                continue
            similarity_dict[image_1_path].append(image_2_path)
            img_1.save(os.path.join(
                "sorted-characters", str(count), image_1_path))
            img_2.save(os.path.join(
                "sorted-characters", str(count), image_2_path))


text:
import { Params } from '@feathersjs/feathers'
import { Base, RestClientParams } from './base'

export class AxiosClient<T = any, D = Partial<T>, P extends Params = RestClientParams> extends Base<T, D, P> {
  request(options: any, params: RestClientParams) {
    const config = Object.assign(
      {
        url: options.url,
        method: options.method,
        data: options.body,
        headers: Object.assign(
          {
            Accept: 'application/json'
          },
          this.options.headers,
          options.headers
        )
      },
      params.connection
    )

    return this.connection
      .request(config)
      .then((res: any) => res.data)
      .catch((error: any) => {
        const response = error.response || error

        throw response instanceof Error ? response : response.data || response
      })
  }
}


text:
require('../com/search');
require('whatwg-fetch');
require('../com/cookie-banner');
const CodeMirror = require('../com/codemirror/CodeMirror');
const NavTree = require('../com/nav-tree');
const ExecutableCode = require('../com/executable-code');
const $ = require('jquery');

$(document).ready(function () {
  new ExecutableCode('.sample');

  CodeMirror.colorize($('.code._highlighted'));

  var html = document.getElementsByTagName('html')[0];

  html.className = html.className.replace('no-js', '');

  // OS detection
  if (navigator.userAgent.indexOf('Linux') > -1)
    html.className += ' os_linux';

  // Browser detection
  if ('chrome' in window)
    html.className += ' ua_chrome';
  else if (navigator.userAgent.toLowerCase().indexOf('firefox') > -1)
    html.className += ' ua_firefox';

  // hack to force :active support in mobile Safari
  document.addEventListener("touchstart", function () {
  }, false);

  $('h1,h2,h3').each(function (element) {
    var id = this.getAttribute("id");
    if (id == null) return;
    var referenceElement = document.createElement("a");
    referenceElement.className = "anchor";
    referenceElement.href = "#" + id;
    this.appendChild(referenceElement);
  });

  var sideTreeElement = document.querySelector('.js-side-tree-nav');
  if (sideTreeElement) {
    new NavTree(sideTreeElement);
  }
});


text:
package org.apache.batik.dom.svg;

import java.util.Locale;
import java.util.ResourceBundle;
import java.util.StringTokenizer;
import java.util.MissingResourceException;

import org.apache.batik.test.AbstractTest;
import org.apache.batik.test.TestReport;
import org.apache.batik.test.DefaultTestReport;

/**
 * This class tests that there is System Id for each public Id
 * in the dtdids.properties resource file.
 *
 * @author Vincent Hardy
 * @author $Id$
 */
public class SystemIdTest extends AbstractTest {
    public static final String ERROR_MISSING_SYSTEM_ID
        = "error.missing.system.id";

    public static final String KEY_MISSING_IDS
        = "key.missing.ids";

    public SystemIdTest() {
    }

    public TestReport runImpl() throws Exception {
        ResourceBundle rb =
            ResourceBundle.getBundle(SAXSVGDocumentFactory.DTDIDS,
                                     Locale.getDefault());
        String dtdids = rb.getString(SAXSVGDocumentFactory.KEY_PUBLIC_IDS);

        StringTokenizer st = new StringTokenizer(dtdids, "-");
        int nIds = st.countTokens();
        String missingIds = "";
        for (int i=0; i<nIds; i++) {
            String publicId = st.nextToken();
            publicId = "-" + publicId.trim();
            System.out.println("Testing public id: " + publicId);
            try {
                rb.getString(SAXSVGDocumentFactory.KEY_SYSTEM_ID
                              + publicId.replace(' ', '_'));
            } catch (MissingResourceException e) {
                missingIds += "[" + publicId + "]  -- ";
            }
        }

        if (!"".equals(missingIds)) {
            DefaultTestReport report = new DefaultTestReport(this);
            report.setErrorCode(ERROR_MISSING_SYSTEM_ID);
            report.addDescriptionEntry(KEY_MISSING_IDS, missingIds);
            report.setPassed(false);
            return report;
        }

        return reportSuccess();
    }
}


text:
<?php

$installer = $this;
/* @var $installer Mage_Sales_Model_Mysql4_Setup */

$installer->startSetup();

$installer->addAttribute('order', 'can_ship_partially', array('type'=>'int'));
$installer->addAttribute('order', 'can_ship_partially_item', array('type'=>'int'));

$installer->endSetup();


text:
<?xml version="1.0" encoding="utf-8"?>
<LinearLayout xmlns:android="http://schemas.android.com/apk/res/android"
    android:orientation="horizontal" android:layout_width="match_parent"
    android:layout_height="wrap_content">

    <TextView
        android:id="@+id/tv_place"
        android:layout_width="wrap_content"
        android:layout_height="wrap_content" />

    <TextView
        android:id="@+id/tv_nickname"
        android:layout_width="0dp"
        android:layout_height="wrap_content"
        android:layout_weight="1"/>

    <TextView
        android:id="@+id/tv_best_word"
        android:layout_width="wrap_content"
        android:layout_height="wrap_content" />

    <TextView
        android:id="@+id/tv_word_count"
        android:layout_width="wrap_content"
        android:layout_height="wrap_content" />

    <TextView
        android:id="@+id/tv_score"
        android:layout_width="wrap_content"
        android:layout_height="wrap_content" />
</LinearLayout>


text:
/*
	1 phrases (passwords) to 512-bits key
*/

#include "C:\Factory\Common\all.h"
#include "C:\Factory\Common\Options\crc.h"
#include "C:\Factory\OpenSource\sha512.h"

#define KEYFILESDIR "C:\\Factory\\tmp\\p1k"

static char *OutputFile;

static void Phrase2Key(char *phrase)
{
	char *file;
	FILE *fp;

	if(!OutputFile)
	{
		file = xcout("[%s].txt", phrase);
		errorCase(!isFairLocalPath(file, strlen(KEYFILESDIR)));
		file = combine_cx(KEYFILESDIR, file);
	}
	else
		file = strx(OutputFile);

	sha512_makeHashLine(phrase);
	sha512_makeHexHash();

	cout("(%02x) %s\n", crc8CheckBlock(sha512_hash, 64), sha512_hexHash);

	removeFileIfExist(file);
	fp = fileOpen(file, "wt");
	writeLine_x(fp, strxl(sha512_hexHash, 64));
	writeLine(fp, sha512_hexHash + 64);
	fileClose(fp);

	memFree(file);
}
int main(int argc, char **argv)
{
	mkdirEx(KEYFILESDIR);

	if(argIs("/O")) // Output file
	{
		OutputFile = nextArg();
	}
	errorCase(hasArgs(2));

	if(hasArgs(1))
	{
		Phrase2Key(nextArg());
	}
	if(!OutputFile)
	{
		execute("START " KEYFILESDIR);
	}
}


text:
package com.linkedin.norbert.network.partitioned.loadbalancer

import com.linkedin.norbert.cluster.Node
import com.linkedin.norbert.network.common.Endpoint
import com.linkedin.norbert.network.partitioned.loadbalancer.gcaware.{GcAwareSimpleConsistentHashedLoadBalancerFactory, GcAwareSimpleConsistentHashedLoadBalancer, GcAwarePartitionedLoadBalancerFactory}
import org.specs.SpecificationWithJUnit
import org.specs.util.WaitFor

/**
 * @author: sishah
 * @date: 06/18/15
 * @version: 1.0
 *
 * TODO sishah: Flesh this out
 */
class GcAwareSimpleConsistentHashedLoadBalancerSpec extends SpecificationWithJUnit with WaitFor {

  val cycleTime = 6000
  val slotTime = 2000

  class TestLBF extends GcAwareSimpleConsistentHashedLoadBalancerFactory[Int](10,
            (id: Int) => HashFunctions.fnv(BigInt(id).toByteArray),
            (str: String) => str.hashCode(), cycleTime, slotTime)

  val loadBalancerFactory = new TestLBF()

  class TestEndpoint(val node: Node, var csr: Boolean) extends Endpoint {
    def canServeRequests = csr

    def setCsr(ncsr: Boolean) {
      csr = ncsr
    }
  }

  def toEndpoints(nodes: Set[Node]): Set[Endpoint] = nodes.map(n => new TestEndpoint(n, true))

  def markUnavailable(endpoints: Set[Endpoint], id: Int) {
    endpoints.foreach { endpoint =>
      if (endpoint.node.id == id) {
        endpoint.asInstanceOf[TestEndpoint].setCsr(false)
      }
    }
  }

  val node1 = new Node(1, "node 1", true, Set(0,1,2), None, None, Some(0))
  val node2 = new Node(2, "node 2", true, Set(3,4,5), None, None, Some(0))
  val node3 = new Node(3, "node 3", true, Set(1,2,3), None, None, Some(1))
  val node4 = new Node(4, "node 4", true, Set(4,5,0), None, None, Some(1))
  val node5 = new Node(5, "node 5", true, Set(1,2,3), None, None, Some(2))
  val node6 = new Node(6, "node 6", true, Set(4,5,0), None, None, Some(2))

  val nodes = Set(node1, node2, node3, node4, node5, node6)

  "GC-aware SimpleConsistentHashed load balancer" should {

    "be sticky WHILE not returning a Node which is currently GCing" in {
      val loadbalancer = loadBalancerFactory.newLoadBalancer(toEndpoints(nodes))
      while(System.currentTimeMillis()%cycleTime != 0){}

      val idCorrespondingToPartition1 = 1210
      val idCorrespondingToPartition4 = 1318

      val possibleNodeSet = scala.collection.mutable.Set(node3,node5)
      val res = loadbalancer.nextNode(idCorrespondingToPartition1)
      res must beSome[Node].which(possibleNodeSet must contain(_))
      possibleNodeSet remove res.get
      val res2 = loadbalancer.nextNode(idCorrespondingToPartition1)
      res2 must be_==(res)

      while(System.currentTimeMillis()%slotTime != 0){}
      val possibleNodeSet2 = scala.collection.mutable.Set(node2,node6)
      val res3 = loadbalancer.nextNode(idCorrespondingToPartition4)
      res3 must beSome[Node].which(possibleNodeSet2 must contain(_))
      possibleNodeSet2 remove res3.get
      val res4 = loadbalancer.nextNode(idCorrespondingToPartition4)
      res4 must be_==(res3)
    }

  }
}


text: exports.filename = module.filename;


text:
sudo pacman -Sy --noconfirm mate-media yay light feh networkmanager dconf termite polkit-gnome sway slurp grim mako swayidle swaylock pamixer

yay -Sy --noconfirm wf-recorder-git


text:
class Gws::Workflow::SearchApproversController < ApplicationController
  include Gws::ApiFilter

  model Gws::User

  prepend_view_path "app/views/workflow/search_approvers"

  before_action :set_group

  private

  def set_group
    if params[:s].present? && params[:s][:group].present?
      @group = @cur_site.descendants.active.find(params[:s][:group])
    else
      @group = @cur_user.groups.active.in_group(@cur_site).first
    end

    @groups = @cur_site.descendants.active.tree_sort(root_name: @cur_site.name)
  end

  def group_ids
    @cur_site.descendants.active.in_group(@group).pluck(:id)
  end

  public

  def index
    @level = params[:level]
    criteria = @model.site(@cur_site).active.search(params[:s])
    criteria = criteria.in(group_ids: group_ids) if @group
    @items = criteria.order_by_title(@cur_site).page(params[:page]).per(50)
  end
end


text:
package org.wso2.carbon.apimgt.core.dao;

import java.util.HashMap;
import java.util.Locale;
import java.util.Map;
import javax.annotation.CheckForNull;

/**
 * Defines the API Types supported in API Manager. These will be stored in the AM_API_TYPES DB table
 */

public enum ApiType {
    STANDARD,
    COMPOSITE;

    private static final Map<String, ApiType> stringToEnum = new HashMap<>();

    static { // Initialize map from constant name to enum constant
        for (ApiType apiType : values()) {
            stringToEnum.put(apiType.toString(), apiType);
        }
    }

    // Returns ApiType for string, or null if string is invalid
    @CheckForNull
    public static ApiType fromString(String symbol) {
        return stringToEnum.get(symbol.toUpperCase(Locale.ENGLISH));
    }
}


text:
title: Better file formats for Impala and quick SQL in Hadoop
author: admin
type: post
date: 2013-10-23T18:03:00+00:00
url: /tutorial-better-file-formats-for-impala-and-quick-sql/
tumblr_gethue_permalink:
  - http://gethue.tumblr.com/post/64879465564/tutorial-better-file-formats-for-impala-and-quick-sql
tumblr_gethue_id:
  - 64879465564
sf_thumbnail_type:
  - none
sf_thumbnail_link_type:
  - link_to_post
sf_detail_type:
  - none
sf_page_title:
  - 1
sf_page_title_style:
  - standard
sf_no_breadcrumbs:
  - 1
sf_page_title_bg:
  - none
sf_page_title_text_style:
  - light
sf_background_image_size:
  - cover
sf_social_sharing:
  - 1
sf_sidebar_config:
  - left-sidebar
sf_left_sidebar:
  - Sidebar-2
sf_right_sidebar:
  - Sidebar-1
sf_caption_position:
  - caption-right
slide_template:
  - default
categories:

---
# Impala File Formats {#docs-internal-guid-798b2644-e679-81ed-508c-e3685cd16a67}

Using the best file format is crucial for getting great performances. This is one reason with JSON is no supported in the [Impala application][1]. Indeed, parsing or retrieving all the text record even for one field would damage the performance badly. Impala is recommending a series of alternative[formats][2].

We show here how to create a Hive table in Avro format containing json data and a table in the new parquet format. We are using the same Yelp data from [Episode 2 of the Season 2][3] of the Hadoop Tutorial series.

## Avro

The first step is to convert our data into JSON with the help of a Pig script. Open up the [Pig Editor][4] and run:

REGISTER piggybank.jar

data = load '/user/hive/warehouse/review/yelp_academic_dataset_review_clean.json'

AS (funny:INT, useful:INT, cool:INT, user_id:CHARARRAY, review_id:CHARARRAY, text:CHARARRAY, business_id:CHARARRAY, stars:INT, date:CHARARRAY, type:CHARARRAY);

data_clean = FILTER data BY business_id IS NOT NULL AND text IS NOT NULL;

STORE data_clean INTO 'impala/reviews_avro'

USING org.apache.pig.piggybank.storage.avro.AvroStorage(

'{

"schema": {

"name": "review",

"type": "record",

"fields": [

{"name":"funny", "type":"int"},

{"name":"useful", "type":"int"},

{"name":"cool", "type":"int"},

{"name":"user_id", "type":"string"}

{"name":"review_id", "type":"string"},

{"name":"text", "type":"string"},

{"name":"business_id", "type":"string"},

{"name":"stars", "type":"int"},

{"name":"date", "type":"string"},

{"name":"type", "type":"string"},

]}

}');

Then, in the [Hive Editor][5] create the table with:

CREATE TABLE review_avro

ROW FORMAT SERDE 'org.apache.hadoop.hive.serde2.avro.AvroSerDe'

STORED AS

inputformat 'org.apache.hadoop.hive.ql.io.avro.AvroContainerInputFormat'

outputformat 'org.apache.hadoop.hive.ql.io.avro.AvroContainerOutputFormat'

LOCATION '/user/romain/impala/reviews_avro'

tblproperties ('avro.schema.literal'='{

"name": "review",

"type": "record",

"fields": [

{"name":"business_id", "type":"string"},

{"name":"cool", "type":"int"},

{"name":"date", "type":"string"},

{"name":"funny", "type":"int"},

{"name":"review_id", "type":"string"},

{"name":"stars", "type":"int"},

{"name":"text", "type":"string"},

{"name":"type", "type":"string"},

{"name":"useful", "type":"int"},

{"name":"user_id", "type":"string"}]}'

);

You can now go back to Impala, and use the table after having refreshed the metadata with:

REFRESH avro_table

## Parquet

Parquet is a new column-oriented binary file format, particularly efficient in Impala. Here is how to create a table from the Impala app:

CREATE TABLE review_parquet LIKE review STORED AS PARQUETFILE;

And then load data:

INSERT OVERWRITE review_parquet SELECT * FROM review;

Take the time to read about the goal of each format and how to enable compression. If you want to know more, the [Impala tuning guide][6] is a good reference too.

As usual feel free to comment on the [hue-user][7] list or [@gethue][8]!

 [1]: http://gethue.tumblr.com/post/62452792255/fast-sql-with-the-impala-query-editor
 [2]: http://www.cloudera.com/content/cloudera-content/cloudera-docs/Impala/latest/Installing-and-Using-Impala/ciiu_file_formats.html
 [3]: http://gethue.tumblr.com/post/60937985689/hadoop-tutorials-ii-2-execute-hive-queries-and
 [4]: http://gethue.tumblr.com/tagged/pig
 [5]: http://gethue.tumblr.com/tagged/hive
 [6]: http://www.cloudera.com/content/cloudera-content/cloudera-docs/Impala/latest/Installing-and-Using-Impala/ciiu_performance.html
 [7]: http://groups.google.com/a/cloudera.org/group/hue-user
 [8]: https://twitter.com/gethue


text:
#region LGPL License

#endregion

#region SVN Version Information

// <file>
//     <license see="http://axiom3d.net/wiki/index.php/license.txt"/>
//     <id value="$Id: Serializer.cs 2940 2012-01-05 12:25:58Z borrillis $"/>
// </file>

#endregion SVN Version Information

#region Namespace Declarations

using System;
using System.IO;
using System.Text;

using Axiom.Core;
using UnityEngine;

#endregion Namespace Declarations

namespace Axiom.Serialization
{
	/// <summary>
	/// Summary description for Serializer.
	/// </summary>
	public class Serializer
	{
		#region Fields

		/// <summary>
		///		Version string of this serializer.
		/// </summary>
		protected string version;

		/// <summary>
		///		Length of the chunk that is currently being processed.
		/// </summary>
		protected int currentChunkLength;

		/// <summary>
		///		Chunk ID + size (short + long).
		/// </summary>
		public const int ChunkOverheadSize = 6;

		#endregion Fields

		#region Constructor

		/// <summary>
		///		Default constructor.
		/// </summary>
		public Serializer()
		{
			// default binary file version
			version = "[Serializer_v1.00]";
		}

		#endregion Constructor

		#region Methods

		/// <summary>
		///		Skips past a particular chunk.
		/// </summary>
		/// <remarks>
		///		Only really used during development, when logic for handling particular chunks is not yet complete.
		/// </remarks>
		protected void IgnoreCurrentChunk( BinaryReader reader )
		{
			Seek( reader, currentChunkLength - ChunkOverheadSize );
		}

		/// <summary>
		///		Reads a specified number of floats and copies them into the destination pointer.
		/// </summary>
		/// <param name="count">Number of values to read.</param>
		/// <param name="dest">Pointer to copy the values into.</param>
		protected void ReadBytes( BinaryReader reader, int count, IntPtr dest )
		{
			// blast the data into the buffer
			unsafe
			{
				byte* pointer = (byte*)dest.ToPointer();
#if !(XBOX || XBOX360 || SILVERLIGHT)
				for( int i = 0; i < count; i++ )
				{
					pointer[ i ] = reader.ReadByte();
				}
#else
				for ( int i =0; i <count ; i+=4 )
				{
						pointer[i + 3] = reader.ReadByte();
						pointer[i + 2] = reader.ReadByte();
						pointer[i+1] = reader.ReadByte();
						pointer[i] = reader.ReadByte();
			}
#endif
			}
		}

		/// <summary>
		///		Writes a specified number of bytes.
		/// </summary>
		/// <param name="count">Number of values to write.</param>
		/// <param name="src">Pointer that holds the values.</param>
		protected void WriteBytes( BinaryWriter writer, int count, IntPtr src )
		{
			unsafe
			{
				byte* pointer = (byte*)src.ToPointer();

				for( int i = 0; i < count; i++ )
				{
					writer.Write( pointer[ i ] );
				}
			}
		}

		/// <summary>
		///		Reads a specified number of floats and copies them into the destination pointer.
		/// </summary>
		/// <param name="count">Number of values to read.</param>
		/// <param name="dest">Pointer to copy the values into.</param>
		protected void ReadFloats( BinaryReader reader, int count, IntPtr dest )
		{
			// blast the data into the buffer
			unsafe
			{
				float* pointer = (float*)dest.ToPointer();

				for( int i = 0; i < count; i++ )
				{
					pointer[ i ] = ReadFloat( reader );
				}
			}
		}

		/// <summary>
		///		Writes a specified number of floats.
		/// </summary>
		/// <param name="count">Number of values to write.</param>
		/// <param name="src">Pointer that holds the values.</param>
		protected void WriteFloats( BinaryWriter writer, int count, IntPtr src )
		{
			unsafe
			{
				float* pointer = (float*)src.ToPointer();

				for( int i = 0; i < count; i++ )
				{
					writer.Write( pointer[ i ] );
				}
			}
		}

		/// <summary>
		///		Reads a specified number of floats and copies them into the destination pointer.
		/// </summary>
		/// <remarks>This overload will also copy the values into the specified destination array.</remarks>
		/// <param name="count">Number of values to read.</param>
		/// <param name="dest">Pointer to copy the values into.</param>
		/// <param name="destArray">A float array that is to have the values copied into it at the same time as 'dest'.</param>
		protected void ReadFloats( BinaryReader reader, int count, IntPtr dest, float[] destArray )
		{
			// blast the data into the buffer
			unsafe
			{
				float* pointer = (float*)dest.ToPointer();

				for( int i = 0; i < count; i++ )
				{
					float val = ReadFloat( reader );
					pointer[ i ] = val;
					destArray[ i ] = val;
				}
			}
		}

		protected bool ReadBool( BinaryReader reader )
		{
			return reader.ReadBoolean();
		}

		protected void WriteBool( BinaryWriter writer, bool val )
		{
			writer.Write( val );
		}

		protected float ReadFloat( BinaryReader reader )
		{
			byte[] inputdata = new byte[4];
			reader.Read( inputdata, 0, 4 );
			// The file format store integral types in little endian order
			if( !BitConverter.IsLittleEndian )
			{
				Array.Reverse( inputdata, 0, 4 );
			}
			return BitConverter.ToSingle( inputdata, 0 );
			return reader.ReadSingle();
		}

		protected void WriteFloat( BinaryWriter writer, float val )
		{
			writer.Write( val );
		}

		protected int ReadInt( BinaryReader reader )
		{
			byte[] inputdata = new byte[4];
			reader.Read( inputdata, 0, 4 );
			// The file format store integral types in little endian order
			if( !BitConverter.IsLittleEndian )
			{
				Array.Reverse( inputdata, 0, 4 );
			}
			return BitConverter.ToInt32( inputdata, 0 );
			return reader.ReadInt32();
		}

		protected void WriteInt( BinaryWriter writer, int val )
		{
			writer.Write( val );
		}

		protected uint ReadUInt( BinaryReader reader )
		{
			byte[] inputdata = new byte[4];
			reader.Read( inputdata, 0, 4 );
			// The file format store integral types in little endian order
			if( !BitConverter.IsLittleEndian )
			{
				Array.Reverse( inputdata, 0, 4 );
			}
			return BitConverter.ToUInt32( inputdata, 0 );

			return reader.ReadUInt32();
		}

		protected void WriteUInt( BinaryWriter writer, uint val )
		{
			writer.Write( val );
		}

		protected long ReadLong( BinaryReader reader )
		{
			byte[] inputdata = new byte[8];
			reader.Read( inputdata, 0, 8 );
			// The file format store integral types in little endian order
			if( !BitConverter.IsLittleEndian )
			{
				Array.Reverse( inputdata, 0, 8 );
			}
			return BitConverter.ToInt64( inputdata, 0 );

			return reader.ReadInt64();
		}

		protected void WriteLong( BinaryWriter writer, long val )
		{
			writer.Write( val );
		}

		protected ulong ReadULong( BinaryReader reader )
		{
			byte[] inputdata = new byte[8];
			reader.Read( inputdata, 0, 8 );
			// The file format store integral types in little endian order
			if( !BitConverter.IsLittleEndian )
			{
				Array.Reverse( inputdata, 0, 8 );
			}
			return BitConverter.ToUInt64( inputdata, 0 );

			return reader.ReadUInt64();
		}

		protected void WriteULong( BinaryWriter writer, ulong val )
		{
			writer.Write( val );
		}

		protected short ReadShort( BinaryReader reader )
		{
			byte[] inputdata = new byte[2];
			reader.Read( inputdata, 0, 2 );
			// The file format store integral types in little endian order
			if( !BitConverter.IsLittleEndian )
			{
				Array.Reverse( inputdata, 0, 2 );
			}
			return BitConverter.ToInt16( inputdata, 0 );

			return reader.ReadInt16();
		}

		protected void WriteShort( BinaryWriter writer, short val )
		{
			writer.Write( val );
		}

		protected ushort ReadUShort( BinaryReader reader )
		{
			byte[] inputdata = new byte[2];
			reader.Read( inputdata, 0, 2 );
			// The file format store integral types in little endian order
			if( !BitConverter.IsLittleEndian )
			{
				Array.Reverse( inputdata, 0, 2 );
			}
			return BitConverter.ToUInt16( inputdata, 0 );

			return reader.ReadUInt16();
		}

		protected void WriteUShort( BinaryWriter writer, ushort val )
		{
			writer.Write( val );
		}

		/// <summary>
		///		Reads a specified number of integers and copies them into the destination pointer.
		/// </summary>
		/// <param name="count">Number of values to read.</param>
		/// <param name="dest">Pointer to copy the values into.</param>
		protected void ReadInts( BinaryReader reader, int count, IntPtr dest )
		{
			// blast the data into the buffer
			unsafe
			{
				int* pointer = (int*)dest.ToPointer();

				for( int i = 0; i < count; i++ )
				{
					pointer[ i ] = ReadInt( reader );
				}
			}
		}

		/// <summary>
		///		Writes a specified number of integers.
		/// </summary>
		/// <param name="count">Number of values to write.</param>
		/// <param name="src">Pointer that holds the values.</param>
		protected void WriteInts( BinaryWriter writer, int count, IntPtr src )
		{
			// blast the data into the buffer
			unsafe
			{
				int* pointer = (int*)src.ToPointer();

				for( int i = 0; i < count; i++ )
				{
					writer.Write( pointer[ i ] );
				}
			}
		}

		/// <summary>
		///		Reads a specified number of shorts and copies them into the destination pointer.
		/// </summary>
		/// <param name="count">Number of values to read.</param>
		/// <param name="dest">Pointer to copy the values into.</param>
		protected void ReadShorts( BinaryReader reader, int count, IntPtr dest )
		{
			// blast the data into the buffer
			unsafe
			{
				short* pointer = (short*)dest.ToPointer();
				for( int i = 0; i < count; i++ )
				{
					pointer[ i ] = ReadShort( reader );
				}
			}
		}

		/// <summary>
		///		Writes a specified number of shorts.
		/// </summary>
		/// <param name="count">Number of values to write.</param>
		/// <param name="src">Pointer that holds the values.</param>
		protected void WriteShorts( BinaryWriter writer, int count, IntPtr src )
		{
			// blast the data into the buffer
			unsafe
			{
				short* pointer = (short*)src.ToPointer();

				for( int i = 0; i < count; i++ )
				{
					writer.Write( pointer[ i ] );
				}
			}
		}

		/// <summary>
		///		Reads from the stream up to the first endline character.
		/// </summary>
		/// <returns>A string formed from characters up to the first '\n' character.</returns>
		protected string ReadString( BinaryReader reader )
		{
			// note: Not using Environment.NewLine here, this character is specifically used in Ogre files.
			return ReadString( reader, '\n' );
		}

		/// <summary>
		///		Writes the string to the stream including the endline character.
		/// </summary>
		protected void WriteString( BinaryWriter writer, string str )
		{
			WriteString( writer, str, '\n' );
		}

		/// <summary>
		///		Reads from the stream up to the specified delimiter character.
		/// </summary>
		/// <param name="delimiter">The character that signals the end of the string.</param>
		/// <returns>A string formed from characters up to the first instance of the specified delimeter.</returns>
		protected string ReadString( BinaryReader reader, char delimiter )
		{
			StringBuilder sb = new StringBuilder();

			char c;

			// sift through each character until we hit the delimiter
			while( ( c = reader.ReadChar() ) != delimiter )
			{
				sb.Append( c );
			}

			// return the accumulated string
			return sb.ToString();
		}

		/// <summary>
		///		Writes the string to the stream including the specified delimiter character.
		/// </summary>
		/// <param name="delimiter">The character that signals the end of the string.</param>
		protected void WriteString( BinaryWriter writer, string str, char delimiter )
		{
			StringBuilder sb = new StringBuilder( str );
			sb.Append( delimiter );
			writer.Write( sb.ToString().ToCharArray() );
		}

		/// <summary>
		///    Reads and returns a Quaternion.
		/// </summary>
		/// <returns></returns>
		protected Quaternion ReadQuat( BinaryReader reader )
		{
			Quaternion quat = new Quaternion();

			quat.x = ReadFloat( reader );
			quat.y = ReadFloat( reader );
			quat.z = ReadFloat( reader );
			quat.w = ReadFloat( reader );

			return quat;
		}

		/// <summary>
		///    Reads and returns a Quaternion.
		/// </summary>
		protected void WriteQuat( BinaryWriter writer, Quaternion quat )
		{
			writer.Write( quat.x );
			writer.Write( quat.y );
			writer.Write( quat.z );
			writer.Write( quat.w );
		}

		/// <summary>
		///    Reads and returns a Vector3 structure.
		/// </summary>
		/// <returns></returns>
		protected Vector3 ReadVector3( BinaryReader reader )
		{
			Vector3 vector = new Vector3();

			vector.x = ReadFloat( reader );
			vector.y = ReadFloat( reader );
			vector.z = ReadFloat( reader );

			return vector;
		}

		/// <summary>
		///    Writes a Vector3 structure.
		/// </summary>
		protected void WriteVector3( BinaryWriter writer, Vector3 vector )
		{
			WriteFloat( writer, vector.x );
			WriteFloat( writer, vector.y );
			WriteFloat( writer, vector.z );
		}

		/// <summary>
		///    Reads and returns a Vector4 structure.
		/// </summary>
		/// <returns></returns>
		protected Vector4 ReadVector4( BinaryReader reader )
		{
			Vector4 vector = new Vector4();

			vector.x = ReadFloat( reader );
			vector.y = ReadFloat( reader );
			vector.z = ReadFloat( reader );
			vector.w = ReadFloat( reader );

			return vector;
		}

		/// <summary>
		///    Writes a Vector4 structure.
		/// </summary>
		protected void WriteVector4( BinaryWriter writer, Vector4 vector )
		{
			WriteFloat( writer, vector.x );
			WriteFloat( writer, vector.y );
			WriteFloat( writer, vector.z );
			WriteFloat( writer, vector.w );
		}

		/// <summary>
		///		Reads a chunk ID and chunk size.
		/// </summary>
		/// <returns>The chunk ID at the current location.</returns>
		protected short ReadFileChunk( BinaryReader reader )
		{
			// get the chunk id
			short id = ReadShort( reader );
#if ( XBOX || XBOX360 )
            if (id == 0)
            {
                reader.BaseStream.Position -= 2;
                id = ReadShort(reader);
            }
#endif
			// read the length for this chunk
			currentChunkLength = ReadInt( reader );

			return id;
		}

		/// <summary>
		///		Writes a chunk ID and chunk size.  This would be more accurately named
		///     WriteChunkHeader, but this name is the counter of ReadChunk.
		/// </summary>
		protected void WriteChunk( BinaryWriter writer, MeshChunkID id, int chunkLength )
		{
			writer.Write( (short)id );
			writer.Write( chunkLength );
		}

		/// <summary>
		///		Writes a chunk ID and chunk size.  This would be more accurately named
		///     WriteChunkHeader, but this name is the counter of ReadChunk.
		/// </summary>
        //protected void WriteChunk( BinaryWriter writer, SkeletonChunkID id, int chunkLength )
        //{
        //    writer.Write( (short)id );
        //    writer.Write( chunkLength );
        //}

		/// <summary>
		///		Reads a file header and checks the version string.
		/// </summary>
		protected void ReadFileHeader( BinaryReader reader )
		{
			short headerID = 0;

			// read the header ID
			headerID = ReadShort( reader );

			// better hope this is the header
			if( headerID == (short)MeshChunkID.Header )
			{
				string fileVersion = ReadString( reader );

				// read the version string
				if( version != fileVersion )
				{
					throw new AxiomException( "Invalid file: version incompatible, file reports {0}, Serializer is version {1}", fileVersion, version );
				}
			}
			else
			{
				throw new AxiomException( "Invalid file: no header found." );
			}
		}

		/// <summary>
		///		Writes a file header and version string.
		/// </summary>
		protected void WriteFileHeader( BinaryWriter writer, string fileVersion )
		{
			writer.Write( (short)MeshChunkID.Header );
			WriteString( writer, fileVersion );
		}

		protected void Seek( BinaryReader reader, long length )
		{
			Seek( reader, length, SeekOrigin.Current );
		}

		/// <summary>
		///		Skips to a particular part of the binary stream.
		/// </summary>
		/// <param name="length">Number of bytes to skip.</param>
		protected void Seek( BinaryReader reader, long length, SeekOrigin origin )
		{
			if( reader.BaseStream.CanSeek )
			{
				reader.BaseStream.Seek( length, origin );
			}
			else
			{
				throw new Exception( "Serializer only supports stream types that CanSeek." );
			}
		}

		protected bool IsEOF( BinaryReader reader )
		{
			// [BUG:1710556]
			// The following commented line would usually work. However, due the fact that we are using PeekChar()
			// Mono and .Net are respecting the encoding type of the file (usually utf-8 ) which is causing this overflow.
			// To avoid this the EOF check is done by comparing the current Position to the Length of the file.
			// return reader.PeekChar() == -1;
			// References :
			//		.Net : http://forums.microsoft.com/MSDN/ShowPost.aspx?PostID=127647&SiteID=1
			//		Mono : http://bugzilla.ximian.com/show_bug.cgi?id=76374
			//			  (google cache ) http://209.85.165.104/search?q=cache:ggELZnLcXpAJ:bugzilla.ximian.com/show_bug.cgi%3Fid%3D76374+An+error+ocurred+in+BinaryReader.PeekChar+()+method+when+execute+the+next&hl=en&ct=clnk&cd=1&gl=us
			return ( reader.BaseStream.Position == reader.BaseStream.Length );
		}

		#endregion Methods
	}
}


text:
import os.path
import unittest

from deduper.wrapper import get_wrapper_for_file

from data import __file__ as data_module

data_dir = os.path.sep.join(data_module.split(os.path.sep)[:-1])

class TestWrapper(unittest.TestCase):
    def setUp(self):
        pass

    def test_hash(self):
        file1 = get_wrapper_for_file(os.path.sep.join((data_dir, 'file1.txt')))
        file2 = get_wrapper_for_file(os.path.sep.join((data_dir, 'file2.txt')))
        dif_file = get_wrapper_for_file(os.path.sep.join((data_dir, 'different_file.txt')))

        self.assertEqual(file1, file2)
        self.assertEqual(file1.hash, file2.hash)
        self.assertNotEqual(file1.hash, None)

        self.assertNotEqual(file1, dif_file)
        self.assertNotEqual(file1.hash, dif_file.hash)
        self.assertEqual(file1.first_block_hash, dif_file.first_block_hash)
        self.assertNotEqual(dif_file.hash, None)

    def test_audio(self):
        file1 = get_wrapper_for_file(os.path.sep.join((data_dir, 'test3.m4a')))
        file2 = get_wrapper_for_file(os.path.sep.join((data_dir, 'test1.m4a')))
        dif_file = get_wrapper_for_file(os.path.sep.join((data_dir, 'test2.m4a')))

        self.assertEqual(file1, file2)
        self.assertEqual(file1.hash, file2.hash)
        self.assertNotEqual(file1.hash, None)

        self.assertNotEqual(file1, dif_file)
        self.assertNotEqual(file1.hash, dif_file.hash)
        self.assertNotEqual(file1.first_block_hash, dif_file.first_block_hash)
        self.assertNotEqual(dif_file.hash, None)

if __name__ == '__main__':
    unittest.main()


text:
/**
 * Module dependencies.
 */

var EventEmitter = require('events').EventEmitter;
var spawn = require('child_process').spawn;
var readlink = require('graceful-readlink').readlinkSync;
var path = require('path');
var dirname = path.dirname;
var basename = path.basename;
var fs = require('fs');

/**
 * Expose the root command.
 */

exports = module.exports = new Command();

/**
 * Expose `Command`.
 */

exports.Command = Command;

/**
 * Expose `Option`.
 */

exports.Option = Option;

/**
 * Initialize a new `Option` with the given `flags` and `description`.
 *
 * @param {String} flags
 * @param {String} description
 * @api public
 */

function Option(flags, description) {
  this.flags = flags;
  this.required = ~flags.indexOf('<');
  this.optional = ~flags.indexOf('[');
  this.bool = !~flags.indexOf('-no-');
  flags = flags.split(/[ ,|]+/);
  if (flags.length > 1 && !/^[[<]/.test(flags[1])) this.short = flags.shift();
  this.long = flags.shift();
  this.description = description || '';
}

/**
 * Return option name.
 *
 * @return {String}
 * @api private
 */

Option.prototype.name = function() {
  return this.long
    .replace('--', '')
    .replace('no-', '');
};

/**
 * Check if `arg` matches the short or long flag.
 *
 * @param {String} arg
 * @return {Boolean}
 * @api private
 */

Option.prototype.is = function(arg) {
  return arg == this.short || arg == this.long;
};

/**
 * Initialize a new `Command`.
 *
 * @param {String} name
 * @api public
 */

function Command(name) {
  this.commands = [];
  this.options = [];
  this._execs = [];
  this._allowUnknownOption = false;
  this._args = [];
  this._name = name;
}

/**
 * Inherit from `EventEmitter.prototype`.
 */

Command.prototype.__proto__ = EventEmitter.prototype;

/**
 * Add command `name`.
 *
 * The `.action()` callback is invoked when the
 * command `name` is specified via __ARGV__,
 * and the remaining arguments are applied to the
 * function for access.
 *
 * When the `name` is "*" an un-matched command
 * will be passed as the first arg, followed by
 * the rest of __ARGV__ remaining.
 *
 * Examples:
 *
 *      program
 *        .version('0.0.1')
 *        .option('-C, --chdir <path>', 'change the working directory')
 *        .option('-c, --config <path>', 'set config path. defaults to ./deploy.conf')
 *        .option('-T, --no-tests', 'ignore test hook')
 *
 *      program
 *        .command('setup')
 *        .description('run remote setup commands')
 *        .action(function() {
 *          console.log('setup');
 *        });
 *
 *      program
 *        .command('exec <cmd>')
 *        .description('run the given remote command')
 *        .action(function(cmd) {
 *          console.log('exec "%s"', cmd);
 *        });
 *
 *      program
 *        .command('teardown <dir> [otherDirs...]')
 *        .description('run teardown commands')
 *        .action(function(dir, otherDirs) {
 *          console.log('dir "%s"', dir);
 *          if (otherDirs) {
 *            otherDirs.forEach(function (oDir) {
 *              console.log('dir "%s"', oDir);
 *            });
 *          }
 *        });
 *
 *      program
 *        .command('*')
 *        .description('deploy the given env')
 *        .action(function(env) {
 *          console.log('deploying "%s"', env);
 *        });
 *
 *      program.parse(process.argv);
  *
 * @param {String} name
 * @param {String} [desc] for git-style sub-commands
 * @return {Command} the new command
 * @api public
 */

Command.prototype.command = function(name, desc, opts) {
  opts = opts || {};
  var args = name.split(/ +/);
  var cmd = new Command(args.shift());

  if (desc) {
    cmd.description(desc);
    this.executables = true;
    this._execs[cmd._name] = true;
  }

  cmd._noHelp = !!opts.noHelp;
  this.commands.push(cmd);
  cmd.parseExpectedArgs(args);
  cmd.parent = this;

  if (desc) return this;
  return cmd;
};

/**
 * Define argument syntax for the top-level command.
 *
 * @api public
 */

Command.prototype.arguments = function (desc) {
  return this.parseExpectedArgs(desc.split(/ +/));
}

/**
 * Add an implicit `help [cmd]` subcommand
 * which invokes `--help` for the given command.
 *
 * @api private
 */

Command.prototype.addImplicitHelpCommand = function() {
  this.command('help [cmd]', 'display help for [cmd]');
};

/**
 * Parse expected `args`.
 *
 * For example `["[type]"]` becomes `[{ required: false, name: 'type' }]`.
 *
 * @param {Array} args
 * @return {Command} for chaining
 * @api public
 */

Command.prototype.parseExpectedArgs = function(args) {
  if (!args.length) return;
  var self = this;
  args.forEach(function(arg) {
    var argDetails = {
      required: false,
      name: '',
      variadic: false
    };

    switch (arg[0]) {
      case '<':
        argDetails.required = true;
        argDetails.name = arg.slice(1, -1);
        break;
      case '[':
        argDetails.name = arg.slice(1, -1);
        break;
    }

    if (argDetails.name.length > 3 && argDetails.name.slice(-3) === '...') {
      argDetails.variadic = true;
      argDetails.name = argDetails.name.slice(0, -3);
    }
    if (argDetails.name) {
      self._args.push(argDetails);
    }
  });
  return this;
};

/**
 * Register callback `fn` for the command.
 *
 * Examples:
 *
 *      program
 *        .command('help')
 *        .description('display verbose help')
 *        .action(function() {
 *           // output help here
 *        });
 *
 * @param {Function} fn
 * @return {Command} for chaining
 * @api public
 */

Command.prototype.action = function(fn) {
  var self = this;
  var listener = function(args, unknown) {
    // Parse any so-far unknown options
    args = args || [];
    unknown = unknown || [];

    var parsed = self.parseOptions(unknown);

    // Output help if necessary
    outputHelpIfNecessary(self, parsed.unknown);

    // If there are still any unknown options, then we simply
    // die, unless someone asked for help, in which case we give it
    // to them, and then we die.
    if (parsed.unknown.length > 0) {
      self.unknownOption(parsed.unknown[0]);
    }

    // Leftover arguments need to be pushed back. Fixes issue #56
    if (parsed.args.length) args = parsed.args.concat(args);

    self._args.forEach(function(arg, i) {
      if (arg.required && null == args[i]) {
        self.missingArgument(arg.name);
      } else if (arg.variadic) {
        if (i !== self._args.length - 1) {
          self.variadicArgNotLast(arg.name);
        }

        args[i] = args.splice(i);
      }
    });

    // Always append ourselves to the end of the arguments,
    // to make sure we match the number of arguments the user
    // expects
    if (self._args.length) {
      args[self._args.length] = self;
    } else {
      args.push(self);
    }

    fn.apply(self, args);
  };
  var parent = this.parent || this;
  var name = parent === this ? '*' : this._name;
  parent.on(name, listener);
  if (this._alias) parent.on(this._alias, listener);
  return this;
};

/**
 * Define option with `flags`, `description` and optional
 * coercion `fn`.
 *
 * The `flags` string should contain both the short and long flags,
 * separated by comma, a pipe or space. The following are all valid
 * all will output this way when `--help` is used.
 *
 *    "-p, --pepper"
 *    "-p|--pepper"
 *    "-p --pepper"
 *
 * Examples:
 *
 *     // simple boolean defaulting to false
 *     program.option('-p, --pepper', 'add pepper');
 *
 *     --pepper
 *     program.pepper
 *     // => Boolean
 *
 *     // simple boolean defaulting to true
 *     program.option('-C, --no-cheese', 'remove cheese');
 *
 *     program.cheese
 *     // => true
 *
 *     --no-cheese
 *     program.cheese
 *     // => false
 *
 *     // required argument
 *     program.option('-C, --chdir <path>', 'change the working directory');
 *
 *     --chdir /tmp
 *     program.chdir
 *     // => "/tmp"
 *
 *     // optional argument
 *     program.option('-c, --cheese [type]', 'add cheese [marble]');
 *
 * @param {String} flags
 * @param {String} description
 * @param {Function|Mixed} fn or default
 * @param {Mixed} defaultValue
 * @return {Command} for chaining
 * @api public
 */

Command.prototype.option = function(flags, description, fn, defaultValue) {
  var self = this
    , option = new Option(flags, description)
    , oname = option.name()
    , name = camelcase(oname);

  // default as 3rd arg
  if (typeof fn != 'function') {
    if (fn instanceof RegExp) {
      var regex = fn;
      fn = function(val, def) {
        var m = regex.exec(val);
        return m ? m[0] : def;
      }
    }
    else {
      defaultValue = fn;
      fn = null;
    }
  }

  // preassign default value only for --no-*, [optional], or <required>
  if (false == option.bool || option.optional || option.required) {
    // when --no-* we make sure default is true
    if (false == option.bool) defaultValue = true;
    // preassign only if we have a default
    if (undefined !== defaultValue) self[name] = defaultValue;
  }

  // register the option
  this.options.push(option);

  // when it's passed assign the value
  // and conditionally invoke the callback
  this.on(oname, function(val) {
    // coercion
    if (null !== val && fn) val = fn(val, undefined === self[name]
      ? defaultValue
      : self[name]);

    // unassigned or bool
    if ('boolean' == typeof self[name] || 'undefined' == typeof self[name]) {
      // if no value, bool true, and we have a default, then use it!
      if (null == val) {
        self[name] = option.bool
          ? defaultValue || true
          : false;
      } else {
        self[name] = val;
      }
    } else if (null !== val) {
      // reassign
      self[name] = val;
    }
  });

  return this;
};

/**
 * Allow unknown options on the command line.
 *
 * @param {Boolean} arg if `true` or omitted, no error will be thrown
 * for unknown options.
 * @api public
 */
Command.prototype.allowUnknownOption = function(arg) {
    this._allowUnknownOption = arguments.length === 0 || arg;
    return this;
};

/**
 * Parse `argv`, settings options and invoking commands when defined.
 *
 * @param {Array} argv
 * @return {Command} for chaining
 * @api public
 */

Command.prototype.parse = function(argv) {
  // implicit help
  if (this.executables) this.addImplicitHelpCommand();

  // store raw args
  this.rawArgs = argv;

  // guess name
  this._name = this._name || basename(argv[1], '.js');

  // github-style sub-commands with no sub-command
  if (this.executables && argv.length < 3) {
    // this user needs help
    argv.push('--help');
  }

  // process argv
  var parsed = this.parseOptions(this.normalize(argv.slice(2)));
  var args = this.args = parsed.args;

  var result = this.parseArgs(this.args, parsed.unknown);

  // executable sub-commands
  var name = result.args[0];
  if (this._execs[name] && typeof this._execs[name] != "function") {
    return this.executeSubCommand(argv, args, parsed.unknown);
  }

  return result;
};

/**
 * Execute a sub-command executable.
 *
 * @param {Array} argv
 * @param {Array} args
 * @param {Array} unknown
 * @api private
 */

Command.prototype.executeSubCommand = function(argv, args, unknown) {
  args = args.concat(unknown);

  if (!args.length) this.help();
  if ('help' == args[0] && 1 == args.length) this.help();

  // <cmd> --help
  if ('help' == args[0]) {
    args[0] = args[1];
    args[1] = '--help';
  }

  // executable
  var f = argv[1];
  // name of the subcommand, link `pm-install`
  var bin = basename(f, '.js') + '-' + args[0];

  // In case of globally installed, get the base dir where executable
  //  subcommand file should be located at
  var baseDir
    , link = readlink(f);

  // when symbolink is relative path
  if (link !== f && link.charAt(0) !== '/') {
    link = path.join(dirname(f), link)
  }
  baseDir = dirname(link);

  // prefer local `./<bin>` to bin in the $PATH
  var localBin = path.join(baseDir, bin);

  // whether bin file is a js script with explicit `.js` extension
  var isExplicitJS = false;
  if (exists(localBin + '.js')) {
    bin = localBin + '.js';
    isExplicitJS = true;
  } else if (exists(localBin)) {
    bin = localBin;
  }

  args = args.slice(1);

  var proc;
  if (process.platform !== 'win32') {
    if (isExplicitJS) {
      args.unshift(localBin);
      // add executable arguments to spawn
      args = (process.execArgv || []).concat(args);

      proc = spawn('node', args, { stdio: 'inherit', customFds: [0, 1, 2] });
    } else {
      proc = spawn(bin, args, { stdio: 'inherit', customFds: [0, 1, 2] });
    }
  } else {
    args.unshift(localBin);
    proc = spawn(process.execPath, args, { stdio: 'inherit'});
  }

  proc.on('close', process.exit.bind(process));
  proc.on('error', function(err) {
    if (err.code == "ENOE


text:
- [Merge Sort](#merge-sort)

## Merge Sort
#### Problem
Given an unsorted array, return the sorted array using a merge sort.

#### Input/Output
```
Input: [1, 5, 2, 4, 3, 6]
Output: [1, 2, 3, 4, 5, 6]
```

#### Explanation
The merge sort uses a divide and conquer method. It first divides the given array until subarrays of length 1. It then merges each subarray with each other. The sort portion occurs during the merge.

When dividing the array, split the array in half similar to a binary search. The trick for the merge is to not iterate through both subarrays but rather just compare the first two values. This is because the subarray will *always* be sorted. The subarray of 1 value will be sorted since it is by itself. When merging two subarrays, the outcome will be a sorted array twice the size of the initial subarray.

The merge sort can be visualized as.
```
Divide:
[1, 5, 2, 4, 3, 6]
[1, 5, 2], [4, 3, 6]
[1, 5], [2], [4, 3], [6]
[1], [5], [2], [4], [3], [6]

Merge:
[1], [5], [2], [4], [3], [6]
[1, 5], [2, 4], [3, 6]
[1, 2, 4, 5], [3, 6]
[1, 2, 3, 4, 5, 6]
```

The divide and merge portions can be done separately or one after the other. As in they can all be divided first and then merged or the first half can be divided which will then be merged before dividing the second half.

The time complexity is O(nlogn) and the space complexity is O(n) where n is the length of the array.

[Implementation](https://github.com/vinnyoodles/algorithms/blob/master/src/sort/mergesort.js)


text:
import datetime as dt
import time

def get_date_begin_by_timestamp(timestamp=None, after_days=0):
    """
    获取时间戳对应的日期的开始时间
    :param timestamp: 时间戳
    :param after_days: 相对于当天后 n 天的时间戳 可以为负数
    :return:
    """
    if timestamp is None:
        timestamp = time.time()
    date_string = time.strftime('%Y-%m-%d', time.localtime(timestamp))
    strut_time = time.strptime(date_string, '%Y-%m-%d')
    return int(time.mktime(strut_time) + constants.SECONDS_OF_DAY * after_days)

def date2timestamp(date_string, date_format='%Y-%m-%d'):
    """
    转换 string 格式为 timestamp
    :param date_string: 要转换的日期字符串
    :param date_format: 日期格式
    :return: timestamp
    """
    # C-level APIs not support key-arguments
    datetime = dt.datetime.strptime(date_string, date_format)
    if not datetime:
        raise ValueError(u'{} for format "{}" not valid'.format(date_string, date_format))
    return int(time.mktime(datetime.timetuple()))

def timestamp2string(timestamp, date_format='%Y-%m-%d'):
    """
    转换 timestamp 为 date_string
    :param timestamp: 要转换的时间戳
    :param date_format: 转换的格式
    :return: date_string
    """
    dtime = dt.datetime.fromtimestamp(timestamp)
    return dtime.strftime(date_format)

def job_date2timestamp(time_string):
    """
    抓取公司下招聘职位时 时间处理(会同时出现'2017-11-12' or '12:01' 两种情况)
    把传入的抓取结果转化为时间戳
    :param time_string:
    :return:
    """
    date, hour_min_seconds = None, None
    if time_string.find(':') != -1:
        hour_min_seconds = time_string
    else:
        date = time_string
    if date is None:
        date = dt.datetime.fromtimestamp(time.time()).strftime('%Y-%m-%d')
    if hour_min_seconds is None:
        hour_min_seconds = '00:00'
    return date2timestamp(date_string=date + ' ' + hour_min_seconds, date_format='%Y-%m-%d %H:%M')


text:
module Perus::Pinger
    class Uptime < Command
        description 'Runs the uptime command and returns the result'
        metric!

        def run
            {uptime: shell('uptime').strip}
        end
    end
end


text:
/*
 @provides _stream_duplex
 @polyfill
 */

module.exports = require('../builtin/_stream_duplex');


text:
/**
 * Created by Andrey Khalzov
 * 31.03.2009
 */
package org.wannatrak.client.exception;

import org.wannatrak.client.Mediator;

public class SubjectInfoNoAccessException extends CheckedException {

    public void throwTo(Mediator mediator) {
        mediator.showSubjectInfoNoAccess();
    }
}


text:
var fs = require('fs');

// Parses a lexicon in JSON or text format
function Lexicon(filename, defaultCategory, defaultCategoryCapitalised) {
  this.lexicon = {};

  if (filename) {
    this.defaultCategory = defaultCategory;
    // Read lexicon
    try {
      var data = fs.readFileSync(filename, 'utf8');
      if (data[0] === "{") {
        // Lexicon is in JSON format
        this.lexicon = JSON.parse(data);
      }
      else {
        // Lexicon is plain text
        this.parseLexicon(data);
      }
      // console.log('Brill_POS_Tagger.read_lexicon: number of lexicon entries read: ' + Object.keys(that.lexicon).length);
    }
    catch (error) {
      console.error(error);
    }
    if (defaultCategory) {
      this.defaultCategory = defaultCategory;
      if (defaultCategoryCapitalised) {
        this.defaultCategoryCapitalised = defaultCategoryCapitalised;
      }
    }
  }
}

// Parses a lexicon in text format: word cat1 cat2 ... catn
Lexicon.prototype.parseLexicon = function(data) {
  // Split into an array of non-empty lines
  var arrayOfLines = data.match(/[^\r\n]+/g);
  this.lexicon = {};
  var that = this;
  arrayOfLines.forEach(function(line) {
    // Split line by whitespace
    var elements = line.trim().split(/\s+/);
    if (elements.length > 0) {
      that.lexicon[elements[0]] = elements.slice(1);
    }
  });
};

// Returns a list of categories for word
Lexicon.prototype.tagWord = function(word) {
  var categories = this.lexicon[word];
  if (!categories) {
    categories = this.lexicon[word.toLowerCase()];
  }
  if (!categories) {
    if (/[A-Z]/.test(word[0]) && this.defaultCategoryCapitalised) {
      // Capitalised
      categories = [this.defaultCategoryCapitalised];
    }
    else {
      // If not found assign default_category
      categories = [this.defaultCategory];
    }
  }
  return(categories);
};

// Adds a word to the lexicon. NB simply replaces the entry
Lexicon.prototype.addWord = function(word, categories) {
  this.lexicon[word] = categories;
};

Lexicon.prototype.prettyPrint = function() {
  var result = "";
  var that = this;
  Object.keys(this.lexicon).forEach(function(token) {
    result += token + "\t";
    that.lexicon[token].forEach(function(cat) {
      result += cat + "\t";
    });
    result += "\n";
  });
  return result;
};

Lexicon.prototype.nrEntries = function() {
  return Object.keys(this.lexicon).length;
};

Lexicon.prototype.setDefaultCategories = function(category, categoryCapitalised) {
  this.defaultCategory = category;
  if (categoryCapitalised) {
    this.defaultCategoryCapitalised = categoryCapitalised;
  }
};

module.exports = Lexicon;


text:
AndroidCacheDemo
================

AndroidCacheDemo