text:
// Copyright (c) 2013, Johns Hopkins University. All rights reserved.
// This software is released under the 2-clause BSD license.
// See /LICENSE.txt

// Travis Wolfe, twolfe18@gmail.com, 30 July 2013

package edu.jhu.hlt.parma.features

import edu.jhu.hlt.parma.util._
import edu.jhu.hlt.parma.types._
import edu.jhu.hlt.parma.feature_interfaces.AlignmentSimilarity
import edu.jhu.hlt.parma.inference.CanonicalMentionFinder
import edu.jhu.hlt.parma.util.SHA1er
import java.security.MessageDigest
import scala.collection.JavaConversions._
import scala.collection.mutable.ArrayBuffer
import redis.clients.jedis._

object RedisStuff {
	val hostname = ParmaConfig.getString("features.ppdb.redis.host")
	val port = ParmaConfig.getInt("features.ppdb.redis.port")

	override def toString: String = "(RedisStuff hostname=%s port=%d)".format(hostname, port)

	/**
	 * redis likes you to give keys that are already hashed, more efficient
	 */
	def makeKey(parts: String*): String = parts.mkString(":")
}

// client
class RedisPPDB extends AlignmentSimilarity {
	import edu.jhu.hlt.parma.features.RedisBloomFilter

	@transient
	private var redis: Jedis = null

	override def name = "RedisPPDB"

	override def setup(calibrateOn: java.util.Collection[DocAlignment]) {
		redis = new Jedis(RedisStuff.hostname, RedisStuff.port)
		if(redis.ping != "PONG")
			throw new RuntimeException("cannot connect to redis! " + RedisStuff.toString)
		redis.select(1)
	}

	private[this] val binarizer = new FixedWidthBinarizer(8, false, -22d, -2d)
	private[this] val emptyFeatures = DVec.rep(0d/*binarizer.agnostic*/, 7)
	private def featuresFromRule(rule: PPDBRule): DVec = {
		implicit def ppdbweight2double(s: String): Double = {
			val v = java.lang.Double.parseDouble(s)
			assert(v >= 0d)
			math.exp(-v)
		}
		val dv = new DVec(
			rule.properties("p(LHS|f)"),
			rule.properties("p(e|LHS)"),
			rule.properties("p(e|f)"),
			rule.properties("p(e|f,LHS)"),
			rule.properties("p(f|LHS)"),
			rule.properties("p(f|e)"),
			rule.properties("p(f|e,LHS)")
		)
		assert(dv.dimension == emptyFeatures.dimension)
		dv
	}

	var q = 0
	var qAtLeastOneHit = 0
	var qAtLeastTwoHits = 0
	val qStep = 5000
	@transient
	lazy val ppdbKeys = RedisBloomFilter.initialize(true)
	private def queryRedis(a: String, b: String, featureName: String, aPOS: String, bPOS: String) {
		val start = System.currentTimeMillis
		val key = a// RedisStuff.makeKey(a, b)
		val maxRHSs = 1000
		val values = if (!ppdbKeys.in(key)) new java.util.ArrayList()
								 else redis.lrange(key, 0, maxRHSs)
		var qi = 0
		val hits = new ArrayBuffer[PPDBRule]
		val hitsPOS = new ArrayBuffer[PPDBRule]
		var sumFeatures: DVec = null
		var maxFeatures: DVec = null
		values.foreach(ruleStr => {
			val rule = PPDBRuleIO.str2rule(ruleStr)
			if(rule.rhs == b) {
				if(q % qStep == 0)
					println("match! q=%d, a=%s, b=%s, rule=%s".format(q, a, b, rule))
				val features = featuresFromRule(rule)
				if(sumFeatures == null) {
					sumFeatures = features.copy
					maxFeatures = features.copy
				} else {
					sumFeatures += features
					maxFeatures.maxEquals(features)
				}
				hits += rule
				if(rule.parent == aPOS && aPOS == bPOS)
					hitsPOS += rule
			}
		})
		if(sumFeatures == null) sumFeatures = emptyFeatures
		for(i <- 0 until sumFeatures.dimension)
			featureIndexer.addUnstable("hits-sum"+i.toString, sumFeatures(i))
		if(maxFeatures == null) maxFeatures = emptyFeatures
		for(i <- 0 until maxFeatures.dimension)
			featureIndexer.addUnstable("hits-max"+i.toString, maxFeatures(i))

		featureIndexer.addUnstable("numHits", hits.size.toDouble / 10d)
		featureIndexer.addUnstable("numHits1", bool2value(hits.size >= 1))
		featureIndexer.addUnstable("numHits2", bool2value(hits.size >= 2))
		featureIndexer.addUnstable("numHits3", bool2value(hits.size >= 4))
		featureIndexer.addUnstable("numHits4", bool2value(hits.size >= 8))
		featureIndexer.addUnstable("numHits5", bool2value(hits.size >= 16))
		featureIndexer.addUnstable("numHits6", bool2value(hits.size >= 32))
		featureIndexer.addUnstable("numHits7", bool2value(hits.size >= 64))

		// filter out PPDB matches that don't match the POS in these mentions
		featureIndexer.addUnstable("numHits-POS", hitsPOS.size.toDouble / 5d)
		featureIndexer.addUnstable("numHits-POS1", bool2value(hitsPOS.size >= 1))
		featureIndexer.addUnstable("numHits-POS2", bool2value(hitsPOS.size >= 2))
		featureIndexer.addUnstable("numHits-POS3", bool2value(hitsPOS.size >= 4))

		// filter based on PPDB score, count how many hits are left
		for(cutoff <- 2d to 6d by 0.2d map { x => x*x }) {
			val c = hits.filter(_.score < cutoff).size
			val cp = hitsPOS.filter(_.score < cutoff).size
			featureIndexer.addUnstable("hits<%.1f".format(cutoff), c / 2d)
			featureIndexer.addUnstable("hits<%.1f-POS".format(cutoff), cp.toDouble)
		}

		val (f1dv, f2dv) =
			if(hits.size > 0) {
				qAtLeastOneHit += 1
				if(hits.size > 1)
					qAtLeastTwoHits += 1
				val l = hits.minBy(_.properties("p(e|f)").toDouble)
				val r = hits.minBy(_.properties("p(f|e)").toDouble)
				val ldv = featuresFromRule(l)
				val rdv = featuresFromRule(r)
				(DVec.max(ldv, rdv), DVec.sum(ldv, rdv))
			}
			else (emptyFeatures, emptyFeatures)
		for(i <- 0 until f1dv.dimension)
			featureIndexer.addUnstable("p(e|f)"+i.toString, f1dv(i))
		for(i <- 0 until f2dv.dimension)
			featureIndexer.addUnstable("p(f|e)"+i.toString, f2dv(i))

		if(q % qStep == 0) {
			val free = Runtime.getRuntime.freeMemory / 1024f / 1024f
			val maxmem = Runtime.getRuntime.maxMemory / 1024f / 1024f
			println("[RedisPPDB featurize] free=%.1f MB, maxMem=%.1f MB".format(free, maxmem))
			println("[RedisPPDB featurize] q=%d qAtLeastOneHit=%d, qAtLeastTwoHits=%d, a=%s, b=%s, %d values processed in %d ms"
				.format(q, qAtLeastOneHit, qAtLeastTwoHits, a, b, values.size, System.currentTimeMillis - start))
		}
		q += 1
	}

	override def featurize(sv: SVec, a: Alignment, report: Document, passage: Document) {

		featureIndexer.start(sv)

		val (rCM, pCM) = CanonicalMentionFinder.canonicalMentions(a, report, passage)
		val rsent = report.getSentence(rCM)
		val psent = passage.getSentence(pCM)

		def pos(t: Token) = "[" + t.getPosTag.toLowerCase + "]"
		def word(t: Token) = t.getWord.toLowerCase

		// [1] mention head word
		val rw = report.getHeadToken(rCM)//.getWord.toLowerCase
		val pw = passage.getHeadToken(pCM)//.getWord.toLowerCase
		if(rw != pw) {
			queryRedis(word(rw), word(pw), "headTok", pos(rw), pos(pw))
			queryRedis(word(pw), word(rw), "headTok", pos(pw), pos(rw))
		}

		// [2] mention head lemma
		//val rMention = report.getMentionSpan(rCM).map(_.getLemma).mkString(" ")
		//val pMention = passage.getMentionSpan(pCM).map(_.getLemma).mkString(" ")
		//queryRedis(rMention, pMention, "mention", addTo)
		//queryRedis(pMention, rMention, "mention", addTo)

		/*
		// [3] 1..k word phrases to the left of both mentions
		val rLeft = rsent.before(rCM)
		val pLeft = psent.before(pCM)
		for(i <- 1 to 3) {
			val rls = rLeft.takeRight(i).map(_.getWord).mkString(" ")
			val pls = pLeft.takeRight(i).map(_.getWord).mkString(" ")
			queryRedis(rls, pls, "left"+i, addTo)
		}

		// [4] 1..k word phrases to the right of both mentions
		val rRight = rsent.after(rCM)
		val pRight = psent.after(pCM)
		for(i <- 1 to 3) {
			val rrs = rRight.take(i).map(_.getWord).mkString(" ")
			val prs = pRight.take(i).map(_.getWord).mkString(" ")
			queryRedis(rrs, prs, "right"+i, addTo)
		}

		// [5] 1..k word phrases on both sides of both mentions
		for(i <- 1 to 3) {
			val r = rsent.slice(rCM.getStartTokenIdx - i, rCM.getEndTokenIdx + i).map(_.getWord).mkString(" ")
			val p = psent.slice(pCM.getStartTokenIdx - i, pCM.getEndTokenIdx + i).map(_.getWord).mkString(" ")
			queryRedis(r, p, "overlap"+i, addTo)
		}
		*/
		featureIndexer.commit
	}

}

object PPDBRuleIO {
	import scala.collection.mutable.HashMap

	// TODO fix this, only tab is supported now
	val sep1 = "\t"
	val sep2 = "\t"
	val eq = "\t"

	def str2rule(redisStr: String): PPDBRule = {
		val ar = redisStr.split("\t")
		val lhs = ar(0)
		val rhs = ar(1)
		val parent = ar(2)
		val score = ar(3).toDouble
		val props = new HashMap[String, String]
		assert(ar.length % 2 == 0)
		for(i <- 4 until ar.length by 2)
			props.update(ar(i), ar(i+1))
		new PPDBRule(lhs, rhs, parent, score, props)
	}

	def validToken(s: String) = s.indexOf(sep1) < 0 && s.indexOf(sep2) < 0 && s.indexOf(eq) < 0

	private def sortedKVs(rule: PPDBRule): String = rule.properties.map(kv => kv._1 + eq + kv._2).toBuffer.sorted.mkString(sep2)

	def rule2str(rule: PPDBRule): String = {
		if(!validToken(rule.lhs))
			println("bad rule.lhs: " + rule.lhs)
		if(!validToken(rule.rhs))
			println("bad rule.rhs: " + rule.rhs)
		assert(validToken(rule.lhs))
		assert(validToken(rule.rhs))
		assert(validToken(rule.parent))
		rule.properties.foreach(kv => {
			if(!validToken(kv._1) || !validToken(kv._2))
				println("bad: " + kv)
			assert(validToken(kv._1))
			assert(validToken(kv._2))
		})
		List(rule.lhs, rule.rhs, rule.parent, rule.score.toString, sortedKVs(rule)).mkString(sep1)
	}
}

class PPDBRule(val lhs: String, val rhs: String, val parent: String, val score: Double, val allProperties: scala.collection.Map[String, String]) {

	val neededKeys = Set("p(LHS|e)", "p(LHS|f)", "p(e|LHS)", "p(e|f)", "p(e|f,LHS)", "p(f|LHS)", "p(f|e)", "p(f|e,LHS)")
	val properties = allProperties.filterKeys(k => neededKeys.contains(k))

	lazy val logProb = neededKeys.map{ k => -allProperties(k).toDouble }.sum
	override def toString = PPDBRuleIO.rule2str(this)
}

// code needed to insert into redis
object RedisPPDBSetup {
	val ppdbKeys = RedisBloomFilter.initialize(true)

	def main(args: Array[String]) {
		if(args.length != 1) {
			println("please provide a parma.config file")
			sys.exit(-1)
		}
		ParmaConfig.load(args(0))
		println("[redis insert] trying to connect to redis at %s on port %d".format(RedisStuff.hostname, RedisStuff.port))
		val redis = new Jedis(RedisStuff.hostname, RedisStuff.port)
		// load redis db 1 instead of db 0
		redis.select(1)
		val f = ParmaConfig.getFile("features.ppdb.redis.file.lexical")
		val reader = FileUtils.getReader(f)

		//println("[redis insert] flushing existing stuff...")
		//val t = redis.multi
		//t.flushDB
		//t.exec
		println("for some reason, cannot flush programatically, do so manually")

		println("[redis insert] inserting stuff...")
		//val pipe = redis.pipelined
		var i = 0
		val start = System.currentTimeMillis
		var last = start
		while(reader.ready) {
			val line = reader.readLine
			val ar = line.split(" \\|\\|\\| ")
			val score = ar(0).toDouble
			val parent = ar(1).toLowerCase
			val leftChild = ar(2).toLowerCase
			val rightChild = ar(3).toLowerCase
			val properties = str2map(ar(ar.length-2))

			val key = leftChild //+"|"+ rightChild
			val value = PPDBRuleIO.rule2str(new PPDBRule(leftChild, rightChild, parent, score, properties))
			//val value = List(rightChild, properties("p(LHS|e)"), properties("p(LHS|f)"),
			//				properties("p(e|LHS)"), properties("p(e|f)"), properties("p(e|f,LHS)"),
			//				properties("p(f|LHS)"), properties("p(f|e)"), properties("p(f|e,LHS)"))
			//println("key = %s, value = [%s]".format(key, value))
			//redis.rpush(key, value: _*)
			redis.rpush(key, value)
			ppdbKeys.set(key)
			//pipe.rpush(key, value)

			val step = 15000
			if(i % step == 0) {
				val now = System.currentTimeMillis
				val recent = step.toDouble / (now-last)
				val avg = i.toDouble / (now-start)
				println("i=%d, %.1f K lines/sec recent, %.1f K lines/sec avg".format(i, recent, avg))
				//println("adding(%d) %s => %s (%.1f %.1f K lines/sec recent/avg)".format(i, key, properties, recent, avg))
				last = now
			}
			i += 1
		}
		reader.close
		ppdbKeys.write
		//pipe.exec

		println("done, added %d rules in %.1f minutes".format(i, (System.currentTimeMillis - start)/(60d*1000d)))
	}

	private def str2map(kvs: String): Map[String, String] = {
		val properties = kvs.split(" ").flatMap(kv => {
			val x = kv.split("=")
			if(x.length != 2) {
				//println("kvs = " + kvs)
				//throw new RuntimeException("x = [%s]".format(x.mkString(", ")))
				Seq()
			}
			else Seq((x(0), x(1)))
		}).toMap

		// get rid of trailing 0s
		properties.mapValues(value => """\.0*$""".r.replaceAllIn(value, ""))
	}

}

/* vim: set noet : */


text:
Details

			Back

			<button class="btn btn-warning btn-lg" ng-click="setEditedListing();">Edit</button>

			<button class="btn btn-danger btn-lg" ng-click="deleteListing();">Delete</button>

 Open

 Perishable

 11/11/22 14:45

 1,000,000,000

 Memo

MEMO TEXT Offal ethical tacos, iPhone migas neutra vice yr freegan green juice williamsburg bicycle
					rights cardigan pork belly.
					Retro poutine irony chartreuse craft beer you probably haven't heard of them. Polaroid vice chicharrones,
					whatever tacos PBR&B
					umami single-origin coffee gluten-free mixtape portland. Readymade vegan pop-up, pug scenester hammock
					8-bit flexitarian master cleanse.

 Hippy Grocery

Address

123 Street NE

Albuquerque

NM

87106

Contact

Kathryn Janeway

(505) 333-4444 ext. 555

Hours

9am-5pm

 Picture

 Location

			<button class="btn btn-success btn-lg btn-block ">Claim Listing</button>

			<button class="btn btn-danger btn-lg btn-block">Cancel Claim</button>

redpajama_set_name: RedPajamaGithub


text:
package org.kongo.kafka.metrics

import java.util.regex.Pattern

import com.yammer.metrics.core.Metric
import com.yammer.metrics.core.MetricName
import com.yammer.metrics.core.MetricPredicate

case class RegexMetricPredicate(include: Option[Pattern], exclude: Option[Pattern]) extends MetricPredicate {
  def matches(metricName: String): Boolean = {
    val includeMatch = include.forall(p => p.matcher(metricName).matches())
    val notExcludeMatch = exclude.forall(p => !p.matcher(metricName).matches())

    includeMatch && notExcludeMatch
  }

  override def matches(name: MetricName, metric: Metric): Boolean = {
    val metricName = MetricFormatter.format(name)
    matches(metricName)
  }
}


text:
package com.android.app.function.eventbus.fragments;

import android.os.Bundle;
import android.view.LayoutInflater;
import android.view.View;
import android.view.ViewGroup;
import android.widget.Toast;

import com.android.app.function.R;
import com.android.app.function.eventbus.bus.MoveToFragmentEvent;
import com.android.app.function.eventbus.bus.NormalEvent;
import com.android.app.function.eventbus.bus.StickyEvent;
import com.android.app.function.eventbus.bus.UpdateActionBarTitleEvent;
import com.android.app.function.eventbus.data.DataObject;

import de.greenrobot.event.EventBus;

/**
 * SecondFragment where user can update the {@link com.kevintan.eventbussample.bus.StickyEvent} or moving to {@link
 * com.kevintan.eventbussample.fragments.ThirdFragment} that enhances the original sample codes.
 *

 * Created by kevintanhongann on 11/27/13.
 *

 * Update:
 *

 * 1. Demonstrate that {@link de.greenrobot.event.EventBus#postSticky(Object)} works including with normal {@link
 * de.greenrobot.event.EventBus#post(Object)}.
 *

 * 2. Update {@link com.kevintan.eventbussample.bus.StickyEvent} and more to third {@link android.app.Fragment} that
 * shows new {@link com.kevintan.eventbussample.bus.StickyEvent}.
 *

 * 3. Post event to update {@link android.app.ActionBar}'s title.
 *

 * 4. Post {@link com.kevintan.eventbussample.bus.MoveToFragmentEvent} that will be handled by{@link
 * com.kevintan.eventbussample.MainActivity} to switch to {@link com.kevintan.eventbussample.fragments.ThirdFragment}.
 *

 * 5. Move calling {@link de.greenrobot.event.EventBus#removeStickyEvent(Class)} on {@link
 * com.kevintan.eventbussample.bus.StickyEvent} into {@link com.kevintan.eventbussample.MainActivity#onDestroy()}.
 *

 * 6. Extends from {@link com.kevintan.eventbussample.fragments.BaseFragment} to demonstrate coexistence of {@link
 * de.greenrobot.event.EventBus#registerSticky(Object)} and {@link de.greenrobot.event.EventBus#register(Object)}.
 *
 * @author Xinyue Zhao
 */
public class SecondFragment extends BaseFragment {
	private static final int LAYOUT = R.layout.eventbus_fragment_second;

	/**
	 * Handler for {@link StickyEvent}
	 *
	 * @param e
	 * 		Event {@link StickyEvent}.
	 */
	public void onEvent(StickyEvent e) {
		Toast.makeText(getActivity(), String.format("Sticky handler: %s", e.getDataObject().toString()), Toast.LENGTH_SHORT).show();
	}
	public void onEvent(NormalEvent e) {
		Toast.makeText(getActivity(), R.string.msg_normal_event, Toast.LENGTH_SHORT).show();
	}

	@Override
	public View onCreateView(LayoutInflater inflater, ViewGroup container,
	                         Bundle savedInstanceState) {
		/*Update ActionBar's title.*/
		EventBus.getDefault().post(new UpdateActionBarTitleEvent(getString(R.string.screen_2)));
		return inflater.inflate(LAYOUT, container, false);
	}

	@Override
	public void onViewStateRestored(Bundle savedInstanceState) {
		super.onViewStateRestored(savedInstanceState);
	}

	@Override
	public void onViewCreated(View view, Bundle savedInstanceState) {
		super.onViewCreated(view, savedInstanceState);
		view.findViewById(R.id.normal_event_btn).setOnClickListener(new View.OnClickListener() {
			@Override
			public void onClick(View v) {
				EventBus.getDefault().post(new NormalEvent());
			}
		});

		view.findViewById(R.id.update_event_btn).setOnClickListener(new View.OnClickListener() {
			@Override
			public void onClick(View v) {
				/*Update the sticky, LOOK the postSticky() runs including post() that onEvent(StickyEvent e) will be called late.*/
				DataObject to = new DataObject("XinyueZ", "dev.xinyue.zhao@gmail.com");
				StickyEvent newEvent = new StickyEvent(to);
				EventBus.getDefault().postSticky(newEvent);
				Toast.makeText(getActivity(), "New Sticky is posted.", Toast.LENGTH_SHORT).show();
			}
		});

		view.findViewById(R.id.next_btn).setOnClickListener(new View.OnClickListener() {
			@Override
			public void onClick(View v) {
				/*Move to 3rd fragment.*/
				EventBus.getDefault().post(new MoveToFragmentEvent(new ThirdFragment()));
			}
		});
	}

	@Override
	protected boolean isStickyAvailable() {
		return true;
	}
}


text:
/*
Depends:
* hmac-sha256.js (Google JS Crypto library)
* jquery.js

Usage:
// Instantiate
var bitstamp = new Bitstamp('clientid', 'apikey', 'apisecret');
// Call with no option parameters
bitstamp.submitRequest(bitstamp.methods.ticker, function(data){console.log(data);} );
// Call with extra parameters
bitstamp.submitRequest(bitstamp.methods.cancelorder, {id: 1}, function(data){console.log(data);} );
// Override REST call
bitstamp.requestFunction = function(xhrParams){ $.ajax(xhrParams) }
*/

Bitstamp = function(client_id, api_key, api_secret) {
  this.auth = {client_id: client_id, api_key: api_key, api_secret: api_secret};
  this.host = 'https://www.bitstamp.net';
  this.methods = {
    ticker: {
        endpoint: '/api/ticker/',
        method: 'GET'
    },
    orderbook: {
        endpoint: '/api/order_book/',
        method: 'GET',
        params: ['group']
    },
    transactions: {
        endpoint: '/api/transactions/',
        method: 'GET',
        params: ['time']
    },
    eurusd: {
        endpoint: '/api/eur_usd/',
        method: 'GET'
    },
    balance: {
        endpoint: '/api/balance/',
        method: 'POST',
        params: ['key', 'signature', 'nonce']
    },
    usertransactions: {
        endpoint: '/api/user_transactions/',
        method: 'POST',
        params: ['key', 'signature', 'nonce', 'offset', 'limit', 'sort']
    },
    openorders: {
        endpoint: '/api/open_orders/',
        method: 'POST',
        params: ['key', 'signature', 'nonce']
    },
    cancelorder: {
        endpoint: '/api/cancel_order/',
        method: 'POST',
        params: ['key', 'signature', 'nonce', 'id']
    },
    orderbuy: {
        endpoint: '/api/buy/',
        method: 'POST',
        params: ['key', 'signature', 'nonce', 'price', 'amount']
    },
    ordersell: {
        endpoint: '/api/sell/',
        method: 'POST',
        params: ['key', 'signature', 'nonce', 'price', 'amount']
    },
    withdrawalrequests: {
        endpoint: '/api/withdrawal_requests/',
        method: 'POST',
        params: ['key', 'signature', 'nonce']
    },
    btcwithdrawal: {
        endpoint: '/api/bitcoin_withdrawal/',
        method: 'POST',
        params: ['key', 'signature', 'nonce', 'amount', 'address']
    },
    btcdepositaddress: {
        endpoint: '/api/bitcoin_deposit_address/',
        method: 'POST',
        params: ['key', 'signature', 'nonce']
    },
    unconfirmedbtc: {
        endpoint: '/api/unconfirmed_btc/',
        method: 'POST',
        params: ['key', 'signature', 'nonce']
    },
    ripplewithdrawal: {
        endpoint: '/api/ripple_withdrawal/',
        method: 'POST',
        params: ['key', 'signature', 'nonce', 'amount', 'address', 'currency']
    },
    rippledepositaddress: {
        endpoint: '/api/ripple_address/',
        method: 'POST',
        params: ['key', 'signature', 'nonce']
    }
  }
}

Bitstamp.prototype.submitRequest = function(bitstampmethod, params, callback) {
  if (typeof params === "function") {
    callback = params;
    params = {};
  }

  if ($.inArray('signature', bitstampmethod.params) > -1 && !('signature' in params) ) {
    console.log('Signature required but not in supplied params');

    unix_timestamp_ms = Math.round(+new Date());
    message = unix_timestamp_ms.toString() + this.auth.client_id + this.auth.api_key;

    var hmac = CryptoJS.algo.HMAC.create(CryptoJS.algo.SHA256, this.auth.api_secret);
    hmac.update(message);
    var hash = hmac.finalize();
    signature = hash.toString().toUpperCase();

    params.key = this.auth.api_key;
    params.signature = signature;
    params.nonce = unix_timestamp_ms;
  }

  for (var param in params) {
    if (typeof params[param] === 'undefined') {
      delete params[param];
    } else {
      params[param] = params[param].toString();
    }
  }

  console.log('Submitting request');

  var that = this;
  this.requestFunction({
    type: bitstampmethod.method,
    url: this.host + bitstampmethod.endpoint,
    data: params,
    success: function(data, textStatus, jqXHR){that.parseResponse(data, callback);},
    error: function(jqXHR, textStatus, errorThrown){that.handleError(textStatus, errorThrown, callback);},
    timeout: 30000,
    dataType: 'json'
  });

  return params;
}

Bitstamp.prototype.requestFunction = function(xhrParams) {
  console.log('requestFunction with params:');
  console.log(xhrParams);

  $.ajax(xhrParams);
}

Bitstamp.prototype.handleError = function(textStatus, errorThrown, callback) {
  console.log('Error returned');
  console.log(errorThrown);

  var data = {};
  data.error = 'Error with request: ' + errorThrown;
  this.parseResponse(data, callback);
}

Bitstamp.prototype.parseResponse = function(response, callback) {
  console.log('Response returned');
  console.log(response);

  var returnval = {};

  if (typeof response === 'object' && 'error' in response) {
    console.log('Error condition');
    var errorstring = '';
    if (typeof response.error === 'string') {
      errorstring = response.error;
    } else {
      for (var key in response.error) {
        errorstring += response.error[key] + "\n";
      }
    }
    returnval.error = errorstring;
  } else {
    returnval.data = response;
  }

  callback(returnval);
}


text:
<!DOCTYPE html>

    <title>pytorch学习10-流量预测实战</title>
    <meta charset="utf-8" />
    <meta http-equiv="X-UA-Compatible" content="IE=edge" />
    <meta name="viewport" content="width=device-width, initial-scale=1" />
    <meta name="keywords" content="pytorch学习10-流量预测实战, pytorch, AI, CodeShurrik" />
    <meta name="description" content="pytorch学习10-流量预测实战, pytorch, AI, 理论结合实际,目前做了一个神经网络在流量预测和告警分析中的应用。利用预测流量与实际流量对比,可以分析出是否发生故障以及预知流量发展趋势,从而提前扩容等等等等。" />
    <meta name="theme-color" content="#2CA6CB"/>
    <link rel="shortcut icon" type="image/x-icon" media="screen" href="http://localhost:4000/favicon.ico" />
    <link rel="canonical" href="http://localhost:4000/2022-09-30/pytorch%E5%AD%A6%E4%B9%A010-%E6%B5%81%E9%87%8F%E9%A2%84%E6%B5%8B%E5%AE%9E%E6%88%98/" />
    <link rel="alternate" type="application/rss+xml" title="CodeShurrik" href="http://localhost:4000/feed.xml"  />

    <link rel="stylesheet" type="text/css" href="http://localhost:4000/static/css/bootstrap.css"/>
    <link rel="stylesheet" href="https://cdn.bootcss.com/octicons/3.5.0/octicons.min.css" >

    <link rel="stylesheet" type="text/css" href="http://localhost:4000/static/css/style.css" />
    <link rel="stylesheet" type="text/css" href="http://localhost:4000/static/css/highlight.css" />
    <link rel="stylesheet" type="text/css" href="http://localhost:4000/static/css/post.css" />

                    <button type="button" class="navbar-toggle" data-toggle="collapse" data-target=".navbar-collapse">

                    </button>
                    CodeShurrik

                          Blog

                              Category

                              Tag

                              FM

                              Monitor

                              About

pytorch学习10-流量预测实战

                 Sep 30, 2022

                 pytorch

                     AI

理论结合实际,目前做了一个神经网络在流量预测和告警分析中的应用。利用预测流量与实际流量对比,可以分析出是否发生故障以及预知流量发展趋势,从而提前扩容等等等等。

以我们某个机房业务为例:

模型建立

考虑影响互联网入口流量的影响因素,基本确定以下6点:

在线设备数量

对象存储利用率

上上一刻流量值

上一刻流量值

当前时间(当天的时间戳)

当前weekday(周几)

后来发现weekday影响不大就去掉了。

采用3层全连接神经网络,输入维度为5,隐含层维度为10,输出为1,其中引入ReLU激活函数,用于增强网络非线性能力。

更新1: 增加一个BN层做标准化(虽然对不深的神经网络没啥用)

class Model(torch.nn.Module):
    def __init__(self):
        super(Model, self).__init__()
        self.linear1 = torch.nn.Linear(5, 10)
        self.bn1 = torch.nn.BatchNorm1d(10)
        self.linear2 = torch.nn.Linear(10, 1)
        # self.linear3 = torch.nn.Linear(2, 1)
        self.relu = torch.nn.ReLU()

    def forward(self, x):
        x = self.relu(self.bn1(self.linear1(x)))
        x = self.relu(self.linear2(x))
        # x = self.sigmoid(self.linear3(x))
        return x

数据采集与处理

其实我感觉深度学习数据集这一块才是重点。。。模型和训练套路都差不多

数据采集用一个定时任务实现,这部分不细说了。

数据处理使用mini-batch,继承Dataset实现。

从数据库中读取数据

按设计模型组装数据

归一化处理

def load_data(thpdataset_lines):
    # 数据载入预处理
    assert isinstance(thpdataset_lines, QuerySet), '原始数据需传入QuerySet'
    # 组装原始数据,列表
    src_data = []
    for thpdataset_line in thpdataset_lines:
        src_data.append(thpdataset_line.device_num)
        src_data.append(thpdataset_line.dxcc_usage)
        src_data.append(thpdataset_line.thp_2)
        src_data.append(thpdataset_line.thp_1)
        src_data.append(thpdataset_line.created_at.hour*3600 +
                        thpdataset_line.created_at.minute*60 + thpdataset_line.created_at.second)  # 当天的时间戳
        src_data.append(thpdataset_line.thp)

    # 列表转换为数组
    np_data = np.array(src_data, dtype=np.float32)
    feature_names = ['device_num', 'dxcc_usage',
                     'thp_2', 'thp_1', 'hour', 'thp']

    # 一维数组数组转换为矩阵, N行6列
    np_data = np_data.reshape(-1, len(feature_names))

    return np_data

def normalize_data(np_data):
    # 计算训练数据集的最大最小值
    max, min = np_data.max(axis=0), np_data.min(axis=0)
    # 记录训练数据的归一化参数,在预测时对数据做反归一化
    np.savez('deeplearn/train_param.npz', max=max, min=min)
    # 对数据进行归一化处理
    norm_data = (np_data - min) / (max - min)

    return norm_data, max, min

class TrainDataset(Dataset):
    def __init__(self, np_data):
        self.x_data = torch.from_numpy(np_data[:, :-1])     # -1列不取
        self.y_data = torch.from_numpy(np_data[:, [-1]])    # 取-1列(矩阵)

        # 获得数据集行数
        self.len = np_data.shape[0]

    # 获得索引方法
    def __getitem__(self, index):
        return self.x_data[index], self.y_data[index]

    # 获得数据集长度
    def __len__(self):
        return self.len

训练

4.按训练数据比例分配数据

5.开始训练

6.最后保存模型参数,通过matplotlib画图

整体代码

import os
import datetime
import torch
import numpy as np
import matplotlib.pyplot as plt
from torch.utils.data import Dataset
from torch.utils.data import DataLoader
from django.db.models.query import QuerySet
from .models import ThpDataset

# plt.style.use('seaborn-whitegrid')
# 显示中文
plt.rcParams['font.sans-serif'] = 'SimHei'
plt.rcParams['axes.unicode_minus'] = False

# 训练参数
EPOCH = 1000
LR = 0.001
RATIO = 0.9

device = torch.device('cuda:0' if torch.cuda.is_available() else 'cpu')

def load_data(thpdataset_lines):
    # 数据载入预处理
    assert isinstance(thpdataset_lines, QuerySet), '原始数据需传入QuerySet'
    # 组装原始数据,列表
    src_data = []
    for thpdataset_line in thpdataset_lines:
        src_data.append(thpdataset_line.device_num)
        src_data.append(thpdataset_line.dxcc_usage)
        src_data.append(thpdataset_line.thp_2)
        src_data.append(thpdataset_line.thp_1)
        src_data.append(thpdataset_line.created_at.hour*3600 +
                        thpdataset_line.created_at.minute*60 + thpdataset_line.created_at.second)  # 当天的时间戳
        src_data.append(thpdataset_line.thp)

    # 列表转换为数组
    np_data = np.array(src_data, dtype=np.float32)
    feature_names = ['device_num', 'dxcc_usage',
                     'thp_2', 'thp_1', 'hour', 'thp']

    # 一维数组数组转换为矩阵, N行6列
    np_data = np_data.reshape(-1, len(feature_names))

    return np_data

def normalize_data(np_data):
    # 计算训练数据集的最大最小值
    max, min = np_data.max(axis=0), np_data.min(axis=0)
    # 记录训练数据的归一化参数,在预测时对数据做反归一化
    np.savez('deeplearn/train_param.npz', max=max, min=min)
    # 对数据进行归一化处理
    norm_data = (np_data - min) / (max - min)

    return norm_data, max, min

class TrainDataset(Dataset):
    def __init__(self, np_data):
        self.x_data = torch.from_numpy(np_data[:, :-1])     # -1列不取
        self.y_data = torch.from_numpy(np_data[:, [-1]])    # 取-1列(矩阵)

        # 获得数据集行数
        self.len = np_data.shape[0]

    # 获得索引方法
    def __getitem__(self, index):
        return self.x_data[index], self.y_data[index]

    # 获得数据集长度
    def __len__(self):
        return self.len

class Model(torch.nn.Module):
    def __init__(self):
        super(Model, self).__init__()
        self.linear1 = torch.nn.Linear(5, 10)
        self.bn1 = torch.nn.BatchNorm1d(10)
        self.linear2 = torch.nn.Linear(10, 1)
        # self.linear3 = torch.nn.Linear(2, 1)   #隐藏层数和大小也是超参数
        self.relu = torch.nn.ReLU()

    def forward(self, x):
        x = self.relu(self.bn1(self.linear1(x)))
        x = self.relu(self.linear2(x))
        # x = self.sigmoid(self.linear3(x))
        return x

# 模型
model = Model()
# 损失函数
criterion = torch.nn.MSELoss()
# 优化器
optimizer = torch.optim.Adam(model.parameters(), lr=LR)
# 学习率调整器
scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(
    optimizer=optimizer, verbose=True)

def train():
    # 读取训练数据并划分训练集和测试集
    now = datetime.datetime.now()
    start = now - datetime.timedelta(days=30)
    thpdataset_lines = ThpDataset.objects.filter(
        created_at__gte=start, created_at__lte=now)

    np_data = load_data(thpdataset_lines)
    norm_data, max, min = normalize_data(np_data)

    offset = int(len(norm_data)*RATIO)
    train_data, test_data = norm_data[:offset], norm_data[offset:]
    train_dataset = TrainDataset(train_data)
    test_dataset = TrainDataset(test_data)

    train_loader = DataLoader(
        dataset=train_dataset, batch_size=64, shuffle=True, num_workers=3, pin_memory=True)
    test_loader = DataLoader(dataset=test_dataset, batch_size=64,
                             shuffle=False, num_workers=3, pin_memory=True)
    x, y = test_dataset[:]
    print(f'[{now}]训练开始,训练集长度为:{len(train_dataset)}')
    # 加载训练模型
    # if os.path.exists('deeplearn/model_param.pkl'):
    #     model.load_state_dict(torch.load('deeplearn/model_param.pkl'))
    train_loss, test_loss = [], []
    model.to(device)
    for epoch in range(EPOCH):
        model.train()
        for i, data in enumerate(train_loader, 0):
            # 准备数据dataloader会将按batch_size返回的数据整合成矩阵加载
            inputs, labels = data
            inputs, labels = inputs.to(device), labels.to(device)
            # 前馈
            y_pred = model(inputs)
            loss = criterion(y_pred, labels)
            # if epoch % 100 == 0:
            #     print(epoch, i, loss.item())
            # 反向传播求梯度
            optimizer.zero_grad()
            loss.backward()
            # 更新
            optimizer.step()
        train_loss.append(loss.item())
        # 调整学习率
        # scheduler.step(loss.item())

        # 测试
        model.eval()
        with torch.no_grad():
            x, y = x.to(device), y.to(device)
            y_pred = model(x)
            loss = criterion(y_pred, y)
            test_loss.append(loss.item())
            # print(epoch, i, train_loss,test_loss)

    # 保存模型参数
    model.to(torch.device('cpu'))
    torch.save(model.state_dict(), 'deeplearn/model_param.pkl')
    print(f'[{datetime.datetime.now()}]模型保存完毕')

    print(f'测试开始,测试集长度为:{len(test_dataset)}')
    # 对测试结果做反归一化处理
    y_pred = y_pred.cpu().numpy() * (max[-1] - min[-1]) + min[-1]
    y = y.cpu().numpy() * (max[-1] - min[-1]) + min[-1]

    y_pred = y_pred*8/1000/1000/1000
    y = y*8/1000/1000/1000
    # print("预测 = ", y_pred)
    # print("实际 = ", y)

    plt.subplot(2, 1, 1)
    plt.grid(True)
    plt.title('训练数据')
    plt.plot(y[:, 0], label='实际流量')
    plt.plot(y_pred[:, 0], label='预测流量')
    plt.ylabel('流量,Gb/s')
    plt.legend()

    plt.subplot(2, 1, 2)
    plt.grid(True)
    plt.plot(train_loss, label='train_loss')
    plt.plot(test_loss, label='test_loss')
    plt.ylabel('误差值')
    plt.legend()
    plt.savefig('deeplearn/test.png')
    plt.close()

最后,模型比较简单,损失值最低只能到这个数量级就不收敛了。还有就是待预测数据会作为下次的训练数据,可能存在过拟合的问题。

                 Toc

                 Tags

                         AI

                 Related

                                pytorch学习15-基于编解码架构的seq2seq优化流量预测

                                pytorch学习14-使用GPU训练

                                pytorch学习13-使用RNN优化流量预测

                                pytorch学习12-循环神经网络基础

                                pytorch学习11-卷积神经网络基础

                                pytorch学习9-多分类问题

                                pytorch学习8-加载数据集

                                pytorch学习7-处理多维特征的输入

                                pytorch学习6-分类问题Logistic回归

                                pytorch学习5-pytorch实现线性回归

                                pytorch学习4-反向传播

                                pytorch学习3-梯度下降算法

                                pytorch学习2-线性模型

                                pytorch学习1-人工智能基础

                                开源版copilot搭建

                © Codeshurrik

                珍爱生命,远离IE。请使用Chrome或Safari,你懂的 :)

                载入天数...载入时分秒...

    <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.16.2/dist/katex.min.css" integrity="sha384-bYdxxUwYipFNohQlHt0bjN/LCpueqWz13HufFEV1SUatKs1cm4L6fFgCi1jT643X" crossorigin="anonymous">


text:
define("gulpfile", ["require", "exports", 'gulp', './tools/utils/yargs', './tools/utils/tasks'], function (require, exports, gulp, yargs_1, tasks_1) {
    "use strict";
    gulp.task('default', function (done) {
        var args = yargs_1.argv.argv;
        console.log('Enviroment is set to:', args.env);
        console.log(args.save + ' library + ' + args.library);
    });
    gulp.task('check.versions', tasks_1.task('check.versions'));
    gulp.task('install', tasks_1.task('install.packages'));
    gulp.task('docs', tasks_1.task('build.docs'));
});
define("Tools/utils/application", ["require", "exports", "Tools/utils/yargs", 'fs'], function (require, exports, yargs_2, fs) {
    "use strict";
    exports.ENVIRONMENTS = {
        DEVELOPMENT: 'dev',
        PRODUCTION: 'prod'
    };
    /*
        retreive enviroment variable that is passed into the system
    */
    function getEnvironment() {
        if (yargs_2.argv.env != null) {
            if (yargs_2.argv.env === exports.ENVIRONMENTS.PRODUCTION) {
                return exports.ENVIRONMENTS.PRODUCTION;
            }
            else {
                //ability to extend more enviroments from here
                return exports.ENVIRONMENTS.DEVELOPMENT;
            }
        }
        else {
            return exports.ENVIRONMENTS.DEVELOPMENT;
        }
    }
    exports.getEnvironment = getEnvironment;
    function getApplicationTitle() {
        return JSON.parse(fs.readFileSync("package.json", "utf8")).description;
    }
    exports.getApplicationTitle = getApplicationTitle;
});
define("Tools/utils/yargs", ["require", "exports", "chalk", "Tools/utils/yargs"], function (require, exports, chalk, yargs) {
    "use strict";
    exports.argv = yargs
        .usage('$0 <command> [options] \n$0 help <command>')
        .alias('b', 'build')
        .command('build', chalk.yellow('S') + 'build', function (yargs) {
        exports.argv = addGlobals(yargs)
            .option("library", {
            alias: "l",
            description: "libary provided"
        })
            .argv;
    })
        .option("env", {
        alias: "e",
        default: 'dev',
        description: "set build enviroment"
    })
        .option("save", {
        alias: "s",
        default: '--save-dev',
        description: "save-dev dependecies"
    })
        .option("library", {
        alias: "l",
        description: "libary provided"
    });
    function addGlobals(argv) {
        argv
            .alias('v', 'version')
            .version(function () { return require('../package').version; })
            .describe('v', 'show version information')
            .count('verbose').alias('v', 'verbose').describe('v', 'Set log verbosity level (-v = info, -vv = warnings, -vvv = debug )')
            .alias('h', 'help')
            .help('help')
            .usage('Usage: $0 -x [num]')
            .showHelpOnFail(false, "Specify --help for available options");
        return argv;
    }
});
define("Tools/config", ["require", "exports", "Tools/utils/application"], function (require, exports, application_1) {
    "use strict";
    exports.ENV = application_1.getEnvironment();
    exports.TOOLS_DIR = 'tools';
    //used by check.versions task
    exports.VERSION_NPM = '2.14.2';
    exports.VERSION_NODE = '4.0.0';
    exports.APP_TITLE = application_1.getApplicationTitle();
    exports.APP_SRC = 'app';
    exports.DOCS_DEST = 'docs';
});
//# sourceMappingURL=app.js.map


text:
/*jshint globalstrict:false, strict:false, maxlen: 500 */
/*global assertEqual, assertTrue, AQL_EXPLAIN, AQL_EXECUTE */

////////////////////////////////////////////////////////////////////////////////
/// @brief tests for optimizer rules
///
/// @file
///
/// DISCLAIMER
///
/// Copyright 2010-2012 triagens GmbH, Cologne, Germany
///
/// Licensed under the Apache License, Version 2.0 (the "License");
/// you may not use this file except in compliance with the License.
/// You may obtain a copy of the License at
///
///     http://www.apache.org/licenses/LICENSE-2.0
///
/// Unless required by applicable law or agreed to in writing, software
/// distributed under the License is distributed on an "AS IS" BASIS,
/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
/// See the License for the specific language governing permissions and
/// limitations under the License.
///
/// Copyright holder is triAGENS GmbH, Cologne, Germany
///
/// @author Jan Steemann
/// @author Copyright 2012, triAGENS GmbH, Cologne, Germany
////////////////////////////////////////////////////////////////////////////////

var jsunity = require("jsunity");
var helper = require("@arangodb/aql-helper");

////////////////////////////////////////////////////////////////////////////////
/// @brief test suite
////////////////////////////////////////////////////////////////////////////////

function optimizerRuleTestSuite () {
  var ruleName = "move-calculations-up";

  // various choices to control the optimizer:
  var paramNone   = { optimizer: { rules: [ "-all" ] } };
  var paramEnabled    = { optimizer: { rules: [ "-all", "+" + ruleName ] } };
  var paramDisabled  = { optimizer: { rules: [ "+all", "-" + ruleName ] } };

  return {

////////////////////////////////////////////////////////////////////////////////
/// @brief set up
////////////////////////////////////////////////////////////////////////////////

    setUp : function () {
    },

////////////////////////////////////////////////////////////////////////////////
/// @brief tear down
////////////////////////////////////////////////////////////////////////////////

    tearDown : function () {
    },

////////////////////////////////////////////////////////////////////////////////
/// @brief test that rule has no effect when explicitly disabled
////////////////////////////////////////////////////////////////////////////////

    testRuleDisabled : function () {
      var queries = [
        "LET a = 1 FOR i IN 1..10 FILTER a == 1 RETURN i",
        "FOR i IN 1..10 LET a = 25 RETURN i",
        "FOR i IN 1..10 LET a = 1 LET b = 2 LET c = 3 FILTER i > 3 LET d = 1 RETURN i"
      ];

      queries.forEach(function(query) {
        var result = AQL_EXPLAIN(query, { }, paramNone);
        assertEqual([ ], result.plan.rules, query);
      });
    },

////////////////////////////////////////////////////////////////////////////////
/// @brief test that rule has no effect
////////////////////////////////////////////////////////////////////////////////

    testRuleNoEffect : function () {
      var queries = [
        "FOR i IN 1..10 FILTER i == 1 RETURN i",
        "FOR i IN 1..10 LET a = 25 + i RETURN i",
        "LET values = 1..10 FOR i IN values LET a = i + 1 FOR j IN values LET b = j + 2 RETURN i",
        "FOR i IN 1..10 FILTER i == 7 COLLECT x = i RETURN x"
      ];

      queries.forEach(function(query) {
        var result = AQL_EXPLAIN(query, { }, paramEnabled);
        assertEqual([ ], result.plan.rules, query);
      });
    },

////////////////////////////////////////////////////////////////////////////////
/// @brief test that rule has an effect
////////////////////////////////////////////////////////////////////////////////

    testRuleHasEffect : function () {
      var queries = [
        "FOR i IN 1..10 LET a = 1 FILTER i == 1 RETURN i",
        "FOR i IN 1..10 LET a = 1 FILTER i == 1 RETURN a",
        "FOR i IN 1..10 FILTER i == 1 LET a = 1 RETURN i",
        "FOR i IN 1..10 LET a = 25 + 7 RETURN i",
        "FOR i IN 1..10 LET a = MIN([ i, 1 ]) LET b = i + 1 RETURN [ a, b ]",
        "FOR i IN 1..10 LET a = RAND() LET b = 25 + i RETURN i",
        "FOR i IN 1..10 LET a = SLEEP(0.1) LET b = i + 1 RETURN b",
        "FOR i IN 1..10 FOR j IN 1..10 LET a = i + 2 RETURN i",
        "FOR i IN 1..10 FILTER i == 7 LET a = i COLLECT x = i RETURN x"
      ];

      queries.forEach(function(query) {
        var result = AQL_EXPLAIN(query, { }, paramEnabled);
        assertEqual([ ruleName ], result.plan.rules, query);
      });
    },

////////////////////////////////////////////////////////////////////////////////
/// @brief test generated plans
////////////////////////////////////////////////////////////////////////////////

    testPlans : function () {
      var plans = [
        [ "FOR i IN 1..10 LET a = 1 RETURN i", [ "SingletonNode", "CalculationNode", "CalculationNode", "EnumerateListNode", "ReturnNode" ] ],
        [ "FOR i IN 1..10 FILTER i == 1 LET a = 1 RETURN i", [ "SingletonNode", "CalculationNode", "CalculationNode", "EnumerateListNode", "CalculationNode", "FilterNode", "ReturnNode"  ] ]
      ];

      plans.forEach(function(plan) {
        var result = AQL_EXPLAIN(plan[0], { }, paramEnabled);
        assertEqual([ ruleName ], result.plan.rules, plan[0]);
        assertEqual(plan[1], helper.getCompactPlan(result).map(function(node) { return node.type; }), plan[0]);
      });
    },

////////////////////////////////////////////////////////////////////////////////
/// @brief test results
////////////////////////////////////////////////////////////////////////////////

    testResults : function () {
      var queries = [
        [ "FOR i IN 1..10 LET a = 1 RETURN i", [ 1, 2, 3, 4, 5, 6, 7, 8, 9, 10 ] ],
        [ "FOR i IN 1..10 LET a = 1 RETURN a", [ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1 ] ],
        [ "FOR i IN 1..10 FILTER i == 1 LET a = 1 RETURN i", [ 1 ] ],
        [ "FOR i IN 1..10 FILTER i == 1 LET a = 1 RETURN a", [ 1 ] ],
      ];

      queries.forEach(function(query) {
        var planDisabled   = AQL_EXPLAIN(query[0], { }, paramDisabled);
        var planEnabled    = AQL_EXPLAIN(query[0], { }, paramEnabled);

        var resultDisabled = AQL_EXECUTE(query[0], { }, paramDisabled);
        var resultEnabled  = AQL_EXECUTE(query[0], { }, paramEnabled);

        assertTrue(planDisabled.plan.rules.indexOf(ruleName) === -1, query[0]);
        assertTrue(planEnabled.plan.rules.indexOf(ruleName) !== -1, query[0]);

        assertEqual(resultDisabled.json, query[1], query[0]);
        assertEqual(resultEnabled.json, query[1], query[0]);
      });
    }

  };
}

////////////////////////////////////////////////////////////////////////////////
/// @brief executes the test suite
////////////////////////////////////////////////////////////////////////////////

jsunity.run(optimizerRuleTestSuite);

return jsunity.done();

// Local Variables:
// mode: outline-minor
// outline-regexp: "^\\(/// @brief\\|/// @addtogroup\\|// --SECTION--\\|/// @page\\|/// @}\\)"
// End:


text:
package com.webfirmframework.wffweb.tag.html.attributewff;

import com.webfirmframework.wffweb.NullValueException;
import com.webfirmframework.wffweb.tag.html.attribute.core.AbstractAttribute;
import com.webfirmframework.wffweb.tag.html.identifier.GlobalAttributable;

/**
 * This is a custom attribute to create an attribute with any name and value.
 *
 * @author WFF
 * @since 12.0.0-beta.5
 */
public class ImmutableCustomAttribute extends AbstractAttribute implements GlobalAttributable {

    private static final long serialVersionUID = 1_0_0L;

    {
        init();
    }

    /**
     * @param attributeName the name for the attribute
     * @since 12.0.0-beta.5
     */
    public ImmutableCustomAttribute(final String attributeName) {
        if (attributeName == null) {
            throw new NullValueException("attributeName can not be null");
        }
        super.setAttributeName(attributeName);
        setAttributeValue(null);
    }

    /**
     * @param attributeName the name for the attribute
     * @param value         the value for the attribute
     * @since 12.0.0-beta.5
     */
    public ImmutableCustomAttribute(final String attributeName, final String value) {
        if (attributeName == null) {
            throw new NullValueException("attributeName can not be null");
        }
        super.setAttributeName(attributeName);
        setAttributeValue(value);
    }

    /**
     * invokes only once per object
     *
     * @author WFF
     * @since 12.0.0-beta.5
     */
    protected void init() {
        // to override and use this method
    }

    /**
     * @return the value
     * @author WFF
     * @since 12.0.0-beta.5
     */
    public String getValue() {
        return getAttributeValue();
    }

}


text:
Use Case 01 - Track Executionss
===================================

Researchers can track code executions inside tales
-----------------------------------------------

Goal
----
Researchers can track code executions inside tales

.. sidebar:: Scenario

    "As a data analyst using R or Jupyter, I want to keep track of my data input files, data output files, and scripts so I can review and compare my runs and the data products they produced."

Summary
-------
The analyst executes a script (or program, function, etc., as appropriate for their computing environment) for which provenance information will be collected.
The script is then executed and the input datasets, derived datasets and execution environment attributes that are to be tracked are automatically determined
and the provenance relationships between these objects is persisted in the tale. Data products such as analysis output, graphical output, etc., that are created by an execution are cached in the tale.

Actors
------
* Investigator
* Client Software

Preconditions
-------------
* The user is working in an instrumented container like Rstudio or Jupyter

Triggers
--------
* Analyst invokes a script or other executable

Post Conditions
---------------
* The analyst has created one or more derived datasets.
* The tale has stored provenance information for the newly created derived datasets.


text:
<?xml version="1.0"?>
<!DOCTYPE Configure PUBLIC "-//Jetty//Configure//EN" "http://www.eclipse.org/jetty/configure.dtd">

<Configure id="wac" class="org.eclipse.jetty.webapp.WebAppContext">
	<New id="mydb" class="org.eclipse.jetty.plus.jndi.Resource">
	<Arg></Arg>
	<Arg>jdbc/jdynametadb</Arg>
	<Arg>
		<New class="org.hsqldb.jdbc.JDBCDataSource">
			<!--Set name="Database">jdbc:hsqldb:file:/home/rainer/projects/db/testdb</Set-->
			<Set name="database">jdbc:hsqldb:hsql://localhost/JDynaMeta</Set>
			<Set name="User">sa</Set>
			<Set name="Password">sa</Set>
		</New>
	</Arg>
	</New>
</Configure>