text:
layout:     post
title:      "mqtt深入浅出系列"
subtitle:   "二、mqtt源码分析"
date:       2017-10-1
author:     "李洋彪"
header-img: "img/post-bg-js-module.jpg"
tags:
    - Android
---

>嗯,没抢到十一的票,明天才能回家,来,写篇博客吧。

### 连接

逐层跟代码,到ClientComms类,该类用于与server通讯,发送和接收mqtt协议消息。

先看connect代码:

	public void connect(MqttConnectOptions options, MqttToken token) throws MqttException {
		final String methodName = "connect";
		synchronized (conLock) {
			if (isDisconnected() && !closePending) {
				//@TRACE 214=state=CONNECTING

				conState = CONNECTING;

				conOptions = options;

                MqttConnect connect = new MqttConnect(client.getClientId(),
                        conOptions.getMqttVersion(),
                        conOptions.isCleanSession(),
                        conOptions.getKeepAliveInterval(),
                        conOptions.getUserName(),
                        conOptions.getPassword(),
                        conOptions.getWillMessage(),
                        conOptions.getWillDestination());

                this.clientState.setKeepAliveSecs(conOptions.getKeepAliveInterval());
                this.clientState.setCleanSession(conOptions.isCleanSession());
                this.clientState.setMaxInflight(conOptions.getMaxInflight());

				tokenStore.open();
				ConnectBG conbg = new ConnectBG(this, token, connect, executorService);
				conbg.start();
			}
			else {
				// @TRACE 207=connect failed: not disconnected {0}
				if (isClosed() || closePending) {
					throw new MqttException(MqttException.REASON_CODE_CLIENT_CLOSED);
				} else if (isConnecting()) {
					throw new MqttException(MqttException.REASON_CODE_CONNECT_IN_PROGRESS);
				} else if (isDisconnecting()) {
					throw new MqttException(MqttException.REASON_CODE_CLIENT_DISCONNECTING);
				} else {
					throw ExceptionHelper.createMqttException(MqttException.REASON_CODE_CLIENT_CONNECTED);
				}
			}
		}
	}

可以看到,无非是:获取之前我们设置的一些参数 --> 然后开启**ConnectBG**连接线程

看ConnectBG的run方法:

	NetworkModule networkModule = networkModules[networkModuleIndex];
	networkModule.start();
	receiver = new CommsReceiver(clientComms, clientState, tokenStore, networkModule.getInputStream());
	receiver.start("MQTT Rec: "+getClient().getClientId(), executorService);
	sender = new CommsSender(clientComms, clientState, tokenStore, networkModule.getOutputStream());
	sender.start("MQTT Snd: "+getClient().getClientId(), executorService);
	callback.start("MQTT Call: "+getClient().getClientId(), executorService);
	internalSend(conPacket, conToken);

这段代码的逻辑是:建立(tcp或者ssl或者WebSocket或者WebSocketSecure)连接 --> 开启接收线程 --> 开启发送线程 --> 发送消息

建立上文中的何种连接方式,是根据serverURI来判断,参考MqttAsyncClient的createNetworkModules方法。

我们分析最常见的tcp连接方式(类TCPNetworkModule的start方法):

	SocketAddress sockaddr = new InetSocketAddress(host, port);
	if (factory instanceof SSLSocketFactory) {
		// SNI support
		Socket tempsocket = new Socket();
		tempsocket.connect(sockaddr, conTimeout*1000);
		socket = ((SSLSocketFactory)factory).createSocket(tempsocket, host, port, true);
	} else {
		socket = factory.createSocket();
		socket.connect(sockaddr, conTimeout*1000);
	}

嗯,最终在这建立的socket连接。

### 如何发送消息

先简单描述:

将message、topic等字段实例化IMqttToken --> 接收线程循环获取msg --> 写流传输

看看具体的代码:

获取msg:

	message = clientState.get();
	if (message != null) {
		//@TRACE 802=network send key={0} msg={1}

		if (message instanceof MqttAck) {
			out.write(message);
			out.flush();
		} else {
			MqttToken token = tokenStore.getToken(message);
			// While quiescing the tokenstore can be cleared so need
			// to check for null for the case where clear occurs
			// while trying to send a message.
			if (token != null) {
				synchronized (token) {
					out.write(message);
					try {
						out.flush();
					} catch (IOException ex) {

					....

### 如何接收消息

读取数据流,再分发通知。

看接收线程的代码:

	while (running && (in != null)) {
			try {
				//@TRACE 852=network read message
				receiving = in.available() > 0;
				MqttWireMessage message = in.readMqttWireMessage();
				receiving = false;

				// instanceof checks if message is null
				if (message instanceof MqttAck) {
					token = tokenStore.getToken(message);
					if (token!=null) {
						synchronized (token) {
							// Ensure the notify processing is done under a lock on the token
							// This ensures that the send processing can complete  before the
							// receive processing starts! ( request and ack and ack processing
							// can occur before request processing is complete if not!
							clientState.notifyReceivedAck((MqttAck)message);
						}
					} else if

					...

### 如何保持长连接的呢

通过闹钟定时任务(AlarmPingSender):

	pendingIntent = PendingIntent.getBroadcast(service, 0, new Intent(action), PendingIntent.FLAG_UPDATE_CURRENT);

    schedule(comms.getKeepAlive());

从AlarmReceiver类onReceive方法入手切入,ClientState的checkForActivity方法有这么一段代码:

    token = new MqttToken(clientComms.getClient().getClientId());
    if(pingCallback != null){
    	token.setActionCallback(pingCallback);
    }
    tokenStore.saveToken(token, pingCommand);
    pendingFlows.insertElementAt(pingCommand, 0);

    nextPingTime = getKeepAlive();

    //Wake sender thread since it may be in wait state (in ClientState.get())
    notifyQueueLock();

原来是pendingFlows队列中插入消息,接收线程收到消息将数据写入流。

### 总结

源码结构清晰,功能很完整,写的还不错的,还是值得一读。

当然也便于自己扩展。


text:
// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
package com.yahoo.schema.processing;

import com.google.common.collect.ImmutableMap;
import com.yahoo.config.model.application.provider.MockFileRegistry;
import com.yahoo.config.model.deploy.TestProperties;
import com.yahoo.config.model.test.MockApplicationPackage;
import com.yahoo.document.DataType;
import com.yahoo.documentmodel.NewDocumentReferenceDataType;
import com.yahoo.schema.DocumentReference;
import com.yahoo.schema.DocumentReferences;
import com.yahoo.schema.Schema;
import com.yahoo.schema.derived.TestableDeployLogger;
import com.yahoo.schema.document.SDDocumentType;
import com.yahoo.schema.document.SDField;
import com.yahoo.schema.document.TemporaryImportedField;
import com.yahoo.schema.document.TemporarySDField;

/*
 * Fixture class used for ImportedFieldsResolverTestCase and AdjustPositionSummaryFieldsTestCase.
 */
public class ParentChildSearchModel {

    public Schema parentSchema;
    public Schema childSchema;

    ParentChildSearchModel() {
        parentSchema = createSearch("parent");
        childSchema = createSearch("child");
    }

    protected Schema createSearch(String name) {
        Schema result = new Schema(name, MockApplicationPackage.createEmpty(), new MockFileRegistry(), new TestableDeployLogger(), new TestProperties());
        result.addDocument(new SDDocumentType(name));
        return result;
    }

    protected static TemporarySDField createField(SDDocumentType repo, String name, DataType dataType, String indexingScript) {
        TemporarySDField result = new TemporarySDField(repo, name, dataType);
        result.parseIndexingScript(indexingScript);
        return result;
    }

    @SuppressWarnings("deprecation")
    protected static SDField createRefField(SDDocumentType repo, String parentType, String fieldName) {
        return new TemporarySDField(repo, fieldName, NewDocumentReferenceDataType.forDocumentName(parentType));
    }

    protected static void addRefField(Schema child, Schema parent, String fieldName) {
        SDField refField = createRefField(child.getDocument(), parent.getName(), fieldName);
        child.getDocument().addField(refField);
        child.getDocument().setDocumentReferences(new DocumentReferences(ImmutableMap.of(refField.getName(),
                new DocumentReference(refField, parent))));
    }

    protected ParentChildSearchModel addImportedField(String fieldName, String referenceFieldName, String targetFieldName) {
        return addImportedField(childSchema, fieldName, referenceFieldName, targetFieldName);
    }

    protected ParentChildSearchModel addImportedField(Schema schema, String fieldName, String referenceFieldName, String targetFieldName) {
        schema.temporaryImportedFields().get().add(new TemporaryImportedField(fieldName, referenceFieldName, targetFieldName));
        return this;
    }
}

redpajama_set_name: RedPajamaGithub


text:
namespace MassTransit
{
    using RabbitMqTransport;

    public static class RabbitMqBasicConsumeContextExtensions
    {
        public static string RoutingKey(this ConsumeContext context)
        {
            return context.TryGetPayload<RabbitMqBasicConsumeContext>(out var consumeContext) ? consumeContext.RoutingKey : string.Empty;
        }

        public static string RoutingKey(this SendContext context)
        {
            return context.TryGetPayload<RabbitMqSendContext>(out var sendContext) ? sendContext.RoutingKey : string.Empty;
        }
    }
}


text:
/*
 * To change this template, choose Tools | Templates
 * and open the template in the editor.
 */
package com.semantic.util;

import java.lang.ref.ReferenceQueue;
import java.lang.ref.SoftReference;
import java.util.AbstractMap;
import java.util.HashMap;
import java.util.LinkedList;
import java.util.Map;
import java.util.Set;

/**
 *
 * http://www.roseindia.net/javatutorials/implementing_softreference_based_hashmap.shtml
 * @param <K>
 * @param <V>
 */
public class SoftHashMap<K, V> extends AbstractMap<K, V> {

    /** The internal HashMap that will hold the SoftReference. */
    private final Map hash = new HashMap();
    /** The number of "hard" references to hold internally. */
    private final int HARD_SIZE;
    /** The FIFO list of hard references, order of last access. */
    private final LinkedList hardCache = new LinkedList();
    /** Reference queue for cleared SoftReference objects. */
    private final ReferenceQueue queue = new ReferenceQueue();

    public SoftHashMap() {
        this(100);
    }

    public SoftHashMap(int hardSize) {
        HARD_SIZE = hardSize;
    }

    @Override
    public V get(Object key) {
        Object result = null;
        // We get the SoftReference represented by that key
        SoftReference soft_ref = (SoftReference) hash.get(key);
        if (soft_ref != null) {
            // From the SoftReference we get the value, which can be
            // null if it was not in the map, or it was removed in
            // the processQueue() method defined below
            result = soft_ref.get();
            if (result == null) {
                // If the value has been garbage collected, remove the
                // entry from the HashMap.
                hash.remove(key);
            } else {
                // We now add this object to the beginning of the hard
                // reference queue.  One reference can occur more than
                // once, because lookups of the FIFO queue are slow, so
                // we don't want to search through it each time to remove
                // duplicates.
                hardCache.addFirst(result);
                if (hardCache.size() > HARD_SIZE) {
                    // Remove the last entry if list longer than HARD_SIZE
                    hardCache.removeLast();
                }
            }
        }
        return (V) result;
    }

    /** We define our own subclass of SoftReference which contains
    not only the value but also the key to make it easier to find
    the entry in the HashMap after it's been garbage collected. */
    private static class SoftValue extends SoftReference {

        private final Object key; // always make data member final

        /** Did you know that an outer class can access private data
        members and methods of an inner class?  I didn't know that!
        I thought it was only the inner class who could access the
        outer class's private information.  An outer class can also
        access private members of an inner class inside its inner
        class. */
        private SoftValue(Object k, Object key, ReferenceQueue q) {
            super(k, q);
            this.key = key;
        }
    }

    /** Here we go through the ReferenceQueue and remove garbage
    collected SoftValue objects from the HashMap by looking them
    up using the SoftValue.key data member. */
    private void processQueue() {
        SoftValue sv;
        while ((sv = (SoftValue) queue.poll()) != null) {
            hash.remove(sv.key); // we can access private data!
        }
    }

    /** Here we put the key, value pair into the HashMap using
    a SoftValue object.
     * @param key
     * @param value
     * @return  */
    @Override
    public Object put(Object key, Object value) {
        processQueue(); // throw out garbage collected values first
        return hash.put(key, new SoftValue(value, key, queue));
    }

    @Override
    public V remove(Object key) {
        processQueue(); // throw out garbage collected values first
        return (V) hash.remove(key);
    }

    @Override
    public void clear() {
        hardCache.clear();
        processQueue(); // throw out garbage collected values
        hash.clear();
    }

    @Override
    public int size() {
        processQueue(); // throw out garbage collected values first
        return hash.size();
    }

    @Override
    public Set entrySet() {
        // no, no, you may NOT do that!!! GRRR
        throw new UnsupportedOperationException();
    }
}


text:
<?xml version="1.0" encoding="UTF-8"?>
<!--
Copyright (C) 2013 TU Dortmund
This file is part of LearnLib, http://www.learnlib.de/.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->
<assembly xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.2"
          xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
          xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.2 http://maven.apache.org/xsd/assembly-1.1.2.xsd">
    <id>jar</id>
    <formats>
        <format>jar</format>
    </formats>

    <includeBaseDirectory>false</includeBaseDirectory>

    <fileSets>
        <fileSet>
            <directory>./</directory>
            <outputDirectory>/</outputDirectory>
            <includes>
                <include>COPYING.txt</include>
                <include>README.md</include>
            </includes>
        </fileSet>
    </fileSets>

    <moduleSets>
        <moduleSet>
            <useAllReactorProjects>true</useAllReactorProjects>
            <includes>
                <include>*:jar:*</include>
            </includes>
            <excludes>
                <exclude>de.learnlib:learnlib-test-support</exclude>
            </excludes>
            <sources>
                <includeModuleDirectory>false</includeModuleDirectory>
                <fileSets>
                    <fileSet>
                        <directory>target/classes</directory>
                        <outputDirectory>/</outputDirectory>
                        <includes>
                            <include>**/*</include>
                        </includes>
                        <excludes>
                            <exclude>*</exclude>
                        </excludes>
                    </fileSet>
                </fileSets>
            </sources>
        </moduleSet>
    </moduleSets>
</assembly>


text:
[![NPM version](http://img.shields.io/npm/v/react-bootstrap-sweetalert.svg)](https://www.npmjs.com/package/react-bootstrap-sweetalert)
[![Downloads](https://img.shields.io/npm/dm/react-bootstrap-sweetalert.svg)](https://www.npmjs.com/package/react-bootstrap-sweetalert)
[![David](https://img.shields.io/david/djorg83/react-bootstrap-sweetalert.svg?maxAge=2592000)](https://github.com/djorg83/react-bootstrap-sweetalert)
[![GitHub issues](https://img.shields.io/github/issues/djorg83/react-bootstrap-sweetalert.svg?maxAge=2592000)](https://github.com/djorg83/react-bootstrap-sweetalert)
[![license](https://img.shields.io/github/license/djorg83/react-bootstrap-sweetalert.svg?maxAge=2592000)](https://github.com/djorg83/react-bootstrap-sweetalert)
[![GitHub stars](https://img.shields.io/github/stars/djorg83/react-bootstrap-sweetalert.svg?style=social&label=Star&maxAge=2592000)](https://github.com/djorg83/react-bootstrap-sweetalert)

[![openbase](https://djorg83.github.io/react-bootstrap-sweetalert/demo/assets/openbase.png) Openbase Dashboard](https://openbase.io/js/react-bootstrap-sweetalert)

[![NPM](https://nodei.co/npm/react-bootstrap-sweetalert.png?downloads=true&stars=true)](https://nodei.co/npm/react-bootstrap-sweetalert/)

# SweetAlert for React/Bootstrap

An awesome replacement for JavaScript's alert.

----

## Support

If you think I've done a good job, consider showing your support by buying me a beer.  Cheers! :beers:

[![Buy me a beer](https://djorg83.github.io/react-bootstrap-sweetalert/demo/assets/bmab.png)](https://www.buymeacoffee.com/djorg83)

----

## Demo & Examples

See the demo with examples of common use cases and a live editor.

:alien: [TAKE ME TO YOUR DEMO](http://djorg83.github.io/react-bootstrap-sweetalert/) :alien:

![Demo GIF](https://djorg83.github.io/react-bootstrap-sweetalert/demo/assets/demo.gif)
----

## Getting Started

### Installation
```
$ npm i react-bootstrap-sweetalert
```

> or

```
$ yarn add react-bootstrap-sweetalert
```

### Import
```
const SweetAlert = require('react-bootstrap-sweetalert');
```
> or

```
import SweetAlert from 'react-bootstrap-sweetalert';
```

----

## Recommended Usage

It is recommended that you keep an alert in your state, and remove it when the `onConfirm` or `onCancel` callbacks are invoked.

You can have stackable alerts by keeping an array of alerts in your data store, and always providing the first alert in
the array as the visible alert.  When an alert is closed, remove it from the store.

See [`examples/redux`](https://github.com/djorg83/react-bootstrap-sweetalert/tree/master/examples/redux) for a working example of how to implement stackable alerts with a Redux store.

----

## Tip: Receiving an input value

If you're using `input` type, the value of the input will be sent to the `onConfirm` callback as the first argument.

``` js
onConfirm={(response) => this.onRecieveInput(response)}
```

## Custom Forms / Using Render Props

If you want to build an alert that re-renders based on external state changes, or simply want to build a custom form,
then you will find the render props pattern to be your best option.

- For re-rendering based on external state changes, use [props.dependencies](#propsdependencies)
- See the `SweetAlertRenderProps` interface in [types.ts](https://github.com/djorg83/react-bootstrap-sweetalert/blob/master/src/types.ts) for some information on the available render props.

```typescript jsx
<SweetAlert
 title={"Uses render props"}
 onConfirm={this.onConfirm}
 onCancel={this.onCancel}
 dependencies={[this.state.firstName, this.state.lastName]}
>
  {(renderProps: SweetAlertRenderProps) => (
    <form>
      Your name is: {this.state.firstName} {this.state.lastName}
      <hr/>
      <input
        type={'text'}
        ref={renderProps.setAutoFocusInputRef}
        className="form-control"
        value={this.state.firstName}
        onKeyDown={renderProps.onEnterKeyDownConfirm}
        onChange={(e) => this.setState({ firstName: e.target.value })}
        placeholder={'First name'}
      />
      <br />
      <input
        type={'text'}
        className="form-control"
        value={this.state.lastName}
        onKeyDown={renderProps.onEnterKeyDownConfirm}
        onChange={(e) => this.setState({ lastName: e.target.value })}
        placeholder={'Last name'}
      />
      <hr/>
    </form>
  )}
</SweetAlert>
```

## Changes in version 5.2

* Added `props.dependencies` that re-renders the alert whenever the provided Array of `dependencies` value changes.
* Added new supported value of `'controlled'` for `props.type`.  If `props.type === 'controlled'` then `props.onConfirm` will return `props.dependencies`.
* Added support for using a function as your alert content/children, aka render props.

For more see [CHANGE_LOG.md](https://github.com/djorg83/react-bootstrap-sweetalert/blob/master/CHANGE_LOG.md)

## Props

- [title](#propstitle) (required)
- [type](#propstype)
- [onConfirm](#propsonconfirm) (required)
- [onCancel](#propsoncancel)
- [customIcon](#propscustomicon)
- [allowEscape](#propsallowescape)
- [closeOnClickOutside](#propscloseonclickoutside)
- [hideOverlay](#propshideoverlay)
- [timeout](#propstimeout)
- [show](#propsshow)
- [dependencies](#propsdependencies)

##### Buttons

- [btnSize](#propsbtnsize)
- [confirmBtnText](#propsconfirmbtntext)
- [confirmBtnBsStyle](#propsconfirmbtnbsstyle)
- [confirmBtnCssClass](#propsconfirmbtncssclass)
- [confirmBtnStyle](#propsconfirmbtnstyle)
- [cancelBtnText](#propscancelbtntext)
- [cancelBtnBsStyle](#propscancelbtnbsstyle)
- [cancelBtnCssClass](#propscancelbtncssclass)
- [cancelBtnStyle](#propscancelbtnstyle)
- [showCloseButton](#propsshowclosebutton)
- [reverseButtons](#propsreversebuttons)
- [customButtons](#propscustombuttons)
- [focusConfirmBtn](#propsfocusconfirmbtn)
- [focusCancelBtn](#propsfocuscancelbtn)
- [closeBtnStyle](#propsclosebtnstyle)
- [showConfirm](#propsshowconfirm)
- [showCancel](#propsshowcancel)
- [disabled](#propsdisabled)

##### Input

- [placeholder](#propsplaceholder)
- [required](#propsrequired)
- [validationMsg](#propsvalidationmsg)
- [validationRegex](#propsvalidationregex)
- [defaultValue](#propsdefaultvalue)
- [inputType](#propsinputtype)

##### Hooks

- [beforeMount](#propsbeforemount)
- [afterMount](#propsaftermount)
- [afterUpdate](#propsafterupdate)
- [beforeUnmount](#propsbeforeunmount)

##### Styling

- [style](#propsstyle)
- [customClass](#propscustomclass)

##### Animations

- [openAnim](#propsopenanim)
- [closeAnim](#propscloseanim)

----

### `props.title`
The text to display for the title. JSX/ReactNode allowed.
- Type: `ReactNode|string`
- Default: `undefined`
----
### `props.onConfirm`
Invoked when user clicks confirm button. Also invoked if user hits ENTER key.
- Type: `(response?: any) => any`
- Default: `undefined`
----
### `props.onCancel`
Invoked when user clicks cancel button. Also invoked if allowEscape is true and user hits ESCAPE key.
- Type: `() => any`
- Default: `undefined`
----
### `props.type`
The type of alert to display.
- Type: `'default'|'info'|'success'|'warning'|'danger'|'error'|'input'|'custom'|'controlled'`
- Default: `'default'`

> NOTE
> - If `props.type === 'controlled'` then `props.onConfirm` will receive `props.dependencies` as its first argument.
> - If `props.type === 'input'` then `props.onConfirm` will receive `props.dependencies` as its first argument.
----
### `props.btnSize`
The type of alert to display.
- Type: `'lg'|'sm'|'xs'`
- Default: `'lg'`
- Allowed values: `'lg'`, `'sm'`, `'xs'`
----
### `props.confirmBtnText`
Content of confirm button, or JSX/ReactNode.
- Type: `ReactNode|string`
- Default: `'OK'`
----
### `props.confirmBtnBsStyle`
Bootstrap style of confirm button.
- Type: `'default'|'primary'|'link'|'info'|'success'|'warning'|'danger'|'secondary'|'outline-{variant}'`
- Default: `'primary'`
----
### `props.confirmBtnCssClass`
CSS class added to confirm button.
- Type: `string`
- Default: `''`
----
### `props.confirmBtnStyle`
Inline style added to confirm button.
- Type: `CSSProperties`
- Default: `{}`
----
### `props.cancelBtnText`
Content of cancel button, or JSX/ReactNode.
- Type: `ReactNode|string`
- Default: `'Cancel'`
----
### `props.cancelBtnBsStyle`
Text of cancel button, or JSX/ReactNode.
- Type: `string`
- Default: `'link'`
- Recommended values: `'default'|'primary'|'link'|'info'|'success'|'warning'|'danger'|'secondary'|'outline-{variant}'`
----
### `props.cancelBtnCssClass`
CSS class added to cancel button.
- Type: `string`
- Default: `''`
----
### `props.cancelBtnStyle`
Inline style added to cancel button.
- Type: `CSSProperties`
- Default: `{}`
----
### `props.showCloseButton`
If set to true, then an X close button will be shown in the top right of the alert.

> NOTE: You must also implement `props.onCancel` in order for this props to work. This is because visibility of the
> component is controlled externally through either `props.show` or by removing the `<SweetAlert />` in your render method.

- Type: `boolean`
- Default: `false`
----
### `props.reverseButtons`
Reverses the order of the Confirm and Cancel buttons. Default positioning is [Cancel] [Confirm]
- Type: `boolean`
- Default: `false`
----
### `props.customButtons`
Custom buttons to use in place of the default Confirm and Cancel buttons. Can render any JSX/ReactNodes here.
- Type: `ReactNode`
- Default: `undefined`
----
### `props.customIcon`
Either a string url for an image to use as the icon, or JSX/ReactNode.
- Type: `ReactNode|string`
- Default: `undefined`
----
### `props.placeholder`
If `props.type` is `'input'`, this is the placeholder for the input field.
- Type: `string`
- Default: `undefined`
----
### `props.show`
If false, the alert will not be rendered.
Warning: Using this option should be a last resort, and is somewhat of an anti-pattern for this library.
The recommended way to control visibility is to only render a `<SweetAlert/>` element when you want one to be displayed,
and remove it when the `onConfirm` or `onCancel` methods are called.
- Type: `boolean`
- Default: `true`
----
### `props.dependencies`
If you have external state that should trigger your alert to re-render it's content, you can provide an `Array` of `dependencies`.
Whenever the dependencies are changed, using `===` comparision, the content of the alert will be re-rendered.
- Type: `any[]`
- Default: `true`

Example
```typescript jsx
const [firstName, setFirstName] = useState('');
const [lastName, setLastName] = useState('');

<SweetAlert dependencies={[firstName, lastName]}>
  <div>
    <h4>Hello {{firstName}} {{lastName}}</h4>
    <input value={firstName} onChange={(e) => setFirstName(e.target.value)} />
    <input value={lastName} onChange={(e) => setLastName(e.target.value)} />
  </div>
</SweetAlert>
```
----
### `props.focusConfirmBtn`
If true the Confirm button will receive focus automatically.  NOTE: Does not apply when `props.type` is `'input'`
- Type: `boolean`
- Default: `true`
----
### `props.focusCancelBtn`
If true the Cancel button will receive focus automatically.  NOTE: Does not apply when `props.type` is `'input'`
- Type: `boolean`
- Default: `false`
----
### `props.required`
If `props.type` is `'input'`, this prop controls whether the input field is required to have a value.
- Type: `boolean`
- Default: `true`
----
### `props.validationMsg`
If `props.type` is `'input'` and `props.required` is `true`, this is the message to display when the user clicks confirm without entering a value.
- Type: `string`
- Default: `'Please enter a response!'`
----
### `props.validationRegex`
If `props.type` is `'input'` and `props.required` is `true`, this Regex is used to validate input value.
- Type: `RegExp`
- Default: `/^.+$/`
----
### `props.defaultValue`
If `props.type` is `'input'`, this is the default value for the input field.
- Type: `number|string`
- Default: `undefined`
----
### `props.inputType`
If `props.type` is `'input'`, this is the default value for the input field.
- Type: `string`
- Default: `'text'`
- Recommended values: `'text'|'password'|'number'|'textarea'`
----
### `props.style`
Style overrides applied to the sweetalert wrapper.
- Type: `CSSProperties`
- Default: `{}`
----
### `props.closeBtnStyle`
Style overrides applied to the X close button.
- Type: `CSSProperties`
- Default: `{}`
----
### `props.customClass`
Custom CSS class applied to the sweetalert wrapper.
- Type: `string`
- Default: `''`
----
### `props.showConfirm`
If `true`, the Confirm button will show.
- Type: `boolean`
- Default: `true`
----
### `props.showCancel`
If `true`, the Cancel button will show.
- Type: `boolean`
- Default: `false`
----
### `props.allowEscape`
If `true`, the `onCancel` function will be invoked when the user hits the `ESCAPE` key.
- Type: `boolean`
- Default: `true`
----
### `props.closeOnClickOutside`
If `true`, the `onCancel` function will be invoked when clicking outside the modal.
- Type: `boolean`
- Default: `true`
----
### `props.hideOverlay`
If `true`, then the modal overlay will not be rendered.
- Type: `boolean`
- Default: `false`
----
### `props.disabled`
If `true`, then the Confirm button will be disabled. (NOTE: This does not effect the `props.allowEscape` behavior.)
If you set disabled to `true` but do not provide an `onCancel` function, then the `disabled` property will not be honored.
- Type: `boolean`
- Default: `false`
----
### `props.beforeMount`
Hook which is invoked at the end of the component `constructor` function.
- Type: `() => any`
- Default: `() => {}`
----
### `props.afterMount`
Hook which is invoked at the end of the `componentDidMount` method.
- Type: `() => any`
- Default: `() => {}`
----
### `props.afterUpdate`
Hook which is invoked at the end of the `componentDidUpdate` method.
- Type: `() => any`
- Default: `() => {}`
----
### `props.beforeUnmount`
Hook which is invoked at the end of the `componentWillUnmount` method.
- Type: `() => any`
- Default: `() => {}`
----
### `props.timeout`
If defined, and greater than `0`, `props.onConfirm` will be invoked to close the alert automatically after the specified number of milliseconds.
- Type: `number`
- Default: `0`
----
### `props.openAnim`
Provide custom show animation or false to have no animation. To specify a custom animation, provide the name of your css animation and duration of the animation in milliseconds.
- Type: `boolean|SweetAlertAnimationProps`
- Default: `{ name: 'showSweetAlert', duration: 300 }`
----
### `props.closeAnim`
Provide custom hide animation or false to have no animation. To specify a custom animation, provide the name of your css animation and duration of the animation in milliseconds. For a simple hide animation you can use `{ name: 'hideSweetAlert', duration: 100 }`
- Type: `boolean|SweetAlertAnimationProps`
- Default: `false`

## Related projects

* [SweetAlert](https://github.com/t4t5/sweetalert)
* [SweetAlert for Android](https://github.com/pedant/sweet-alert-dialog)
* [SweetAlert for Bootstrap](https://github.com/lipis/bootstrap-sweetalert)
* [SweetAlert for AngularJS](https://github.com/oitozero/ngSweetAlert)
* [SweetAlert for RubyOnRails](https://github.com/sharshenov/sweetalert-rails)

## Development

``` bash
$ yarn demo && open http://localhost:3000
```


text:
[![license](https://img.shields.io/github/license/mashape/apistatus.svg?maxAge=2592000)](https://github.com/Liebeck/OnlineParticipationDatasets/blob/master/LICENSE)

This projects aims to share text content from online participation processes by sharing crawlers instead of the text contents themselves which might not be possible due to legal reasons. The text contents of the individual online participation plattforms will be downloaded via [Scrapy](https://scrapy.org/) into a JSON file.

## Dependencies
* [docker](https://www.docker.com/)

## How to setup the project:
This will install all required dependencies:
``` bash
make build
```

## How to start the project:
``` bash
make run
scrapy crawl <dataset>
```

## Supported datasets:

| Dataset name | Link | # Suggestions | # Comments | # Total | Crawl time | Command|
|---|---|---|---|---|---|---|
| Bonn 2011    | [~~bonn-packts-an-2011.de~~](http://bonn-packts-an-2011.de/www.bonn-packts-an.de/dito/forumc0d2.html) [2](#myfootnote2)                                     | 1015 | 8903 | 9918 | 50 seconds | scrapy crawl bonn2011 |
| Bonn 2015/2016 |[bonn-macht-mit 2015/2016](https://www.bonn-macht-mit.de/dialog/bürgerbeteiligung-am-haushalt-20152016/bhh/online-diskussion) | 335 | 2937 | 3271 | 27 seconds | scrapy crawl bonn2015 |
| Bonn 2017/2018 |[bonn-macht-mit 2017/2018](https://www.bonn-macht-mit.de/node/871) | 55 | 109 | 164 | 5 seconds | scrapy crawl bonn2017 |
| Bonn 2019/2020 |[bonn-macht-mit 2019/2020](https://www.bonn-macht-mit.de/node/2900) | 232 | 600 | 832 | 1 minute | scrapy crawl bonn2019 |
| Braunkohle[1](#myfootnote1)| [~~leitentscheidung-braunkohle.nrw~~](https://www.leitentscheidung-braunkohle.nrw/perspektiven/de/home/beteiligen/draftbill/47589/chap/5#chapter) [2](#myfootnote2)| 7 | 1296 | 1296 | 32 minutes | scrapy crawl braunkohle|
| Bürgerbudget Wuppertal |[buergerbudget.wuppertal.de](https://buergerbudget.wuppertal.de/cb/t711bwqTXj3GSGiEVwa3li3YZDqvq4pL?type=phase1) | 261 | | | 4 minutes | scrapy crawl wuppertal2017 |
| Köln 2012 | [buergerhaushalt.stadt-koeln.de/2012/diskussion](https://buergerhaushalt.stadt-koeln.de/2012/diskussion) | 594 | 1879 | 2473 | 18 minutes | scrapy crawl koeln2012|
| Köln 2013 | [buergerhaushalt.stadt-koeln.de/2013/buergervorschlaege](https://buergerhaushalt.stadt-koeln.de/2013/buergervorschlaege?&sort_bef_combine=php+ASC) | 592 | 3095 | 3687 | 5 minutes | scrapy crawl koeln2013|
| Köln 2015 | [buergerhaushalt.stadt-koeln.de/2015/buergervorschlaege](https://buergerhaushalt.stadt-koeln.de/2015/buergervorschlaege?&sort_bef_combine=php+ASC) | 631 | 1855 | 2486 | 10 minutes | scrapy crawl koeln2015|
| Köln 2016 | [buergerhaushalt.stadt-koeln.de/2016/buergervorschlaege](https://buergerhaushalt.stadt-koeln.de/2016/buergervorschlaege?&sort_bef_combine=php+ASC) | 827 | 1314 | 2141 | 9 minutes | scrapy crawl koeln2016|
| Leitbild Bad Godesberg | [bonn-macht-mit.de/dialoge/onlinebeteiligung-zum-leitbild-bad-godesberg](https://www.bonn-macht-mit.de/dialoge/onlinebeteiligung-zum-leitbild-bad-godesberg) | 556 | 698 | 1254 | 2 minutes | scrapy crawl badgodesberg|
| Mängelmelder Braunschweig | [mitreden.braunschweig.de/node/1358](https://www.mitreden.braunschweig.de/node/1358) | ≥ 3220 | | | 34 minutes | scrapy crawl maengelmelder-braunschweig|
| Nahverkehrsplan Ulm | [zukunftsstadt-ulm.de/dialog](https://www.zukunftsstadt-ulm.de/dialog/hier-konnten-sie-mit-uns-zu-den-einzelnen-bus-und-bahnlinien-diskutieren) | 498 | 619 | 1117 | 1 minute | scrapy crawl nahverkehrsplan-ulm |
| Raddialog Bonn | [raddialog.bonn.de/dialoge](https://www.raddialog.bonn.de/dialoge/bonner-rad-dialog?sort_bef_combine=created%20ASC) | 2331 | 2425 | 4756 | 16 minutes | scrapy crawl raddialog-bonn|
| Raddialog Koeln | [raddialog-ehrenfeld.koeln.de/dialoge](http://www.raddialog-ehrenfeld.koeln/dialoge/ehrenfelder-raddialog?sort_bef_combine=created%20ASC) | 378 | 277 | 655 | 2 minutes | scrapy crawl raddialog-koeln|
| Raddialog Moers | [raddialog.moers.de/dialoge](https://www.raddialog.moers.de/node/1384?sort_bef_combine=created%20ASC) | 463 | 300 | 763 | 3 minutes | scrapy crawl raddialog-moers|

1: Braunkohle-spider uses javascript to load content and needs to wait for server responses. This results in long crawltimes.Braunkohle-site only provides 7 suggestions.

2: Offline


text: <meta http-equiv="refresh" content="0; url=Effective_002dTarget-Keywords.html#arm%5fneonv2%5fok">


text:
from pypassivedns.provider import Provider
from pypassivedns.pypassivedns import PDNSResult
import json
import re
import datetime
import time

class DNSDB(Provider):
    NAME = "DNSDB"
    CONF = "dnsdb"
    OPTL = "d"

    def __init__(self, options={}):
        self.debug = options.get("debug", False)
        self.apikey = options.get("apikey", None)
        self.url = options.get("url", "https://api.dnsdb.info/lookup")

    def query(self, query, limit=None):
        url = None
        if re.match("[\d\.]+$", query):
            url = "%s/rdata/ip/%s" % (self.url, query)
        else:
            url = "%s/rrset/name/%s" % (self.url, query)
        params = {}
        if limit:
            params["limit"] = limit
        start_time = time.time()

        data = Provider.get_json(url, "GET", params, { "X-API-Key" : self.apikey }, True)
        response_time = time.time() - start_time
        return self._data_to_records(query, data, response_time)

    @staticmethod
    def _data_to_records(query, data, response_time):
        recs = []
        for row in data:
            answers = row.get('rdata')
            if type(answers) == str:
                answers = [answers]
            query = row.get('rrname', '')
            query = re.sub('\.$', '', query)
            rrtype = row.get('rrtype', 'A')
            firstseen = datetime.datetime.utcfromtimestamp(int(row.get('time_first', '0')))
            lastseen = datetime.datetime.utcfromtimestamp(int(row.get('time_last', '0')))
            count = row.get('count', '0')
            for answer in answers:
                recs.append(
                    PDNSResult(DNSDB.NAME, response_time, query, answer, rrtype, 0, firstseen, lastseen, count)
                )
        return recs


text:
# parch

[![Build Status](https://travis-ci.org/dylanfoster/parch.svg?branch=master)](https://travis-ci.org/dylanfoster/parch)
[![Coverage Status](https://coveralls.io/repos/github/dylanfoster/parch/badge.svg?branch=develop)](https://coveralls.io/github/dylanfoster/parch?branch=develop)
[![bitHound Overall Score](https://www.bithound.io/github/dylanfoster/parch/badges/score.svg)](https://www.bithound.io/github/dylanfoster/parch)

> [Restify](http://restify.com/) + [Sequelize](http://docs.sequelizejs.com/en/latest/)

Parch combines restify and sequelize to bring you a powerful yet easy setup for
your API

## Installation

npm

```bash
npm install --save parch
```

yarn

```bash
yarn add --save parch
```

## Usage

- [Application](#application)
- [Router](#router)
  - [#resource](#resource)
  - [#route](#route)
  - [#namespace](#namespace)
- [Controller](#controller)
  - [lifecycle hooks](#controller-lifecycle)
  - [hooks](#controller-hooks) **deprecated**
  - [nested controllers](#nested-controllers)
- [Model](#model)
- [Serializers](#serializers)
- [Associations](#associations-wip)
- [Authentication](#authentication-and-authorization)
- [Application Initializers](#application-initializers)
- [Logging](#logging)
- [Error handling](#error-handling-and-responses)
  - [Error Codes](#errors)
  - [Responses](#responses)

### Application

For a full list of available options [see below](#options)

```javascript
const parch = require("parch");

// define your app
const app = new parch.Application({
  authentication: {
    secretKey: "ssshhh",
    unauthenticated: [/\/posts[\s\S]*/, "/users/resetPassword"]
  },

  controllers: {
    dir: path.resolve(__dirname, "controllers")
  },

  database: {
    connection: {
      username: "postgres",
      password: "postgres",
      database: "postgres",
      host:  "localhost",
      dialect: "postgres",
      logging: false
    },

    models: {
      dir: path.resolve(__dirname, "models")
    }
  },

  initializers: {
    dir: path.resolve(__dirname, "controllers")
  },

  logging: {
    dir: path.resolve(__dirname, 'logs'),
    logger: Bunyan.createLogger(),
    serializers: {
      req(req) {
        return {
          url: req.url
        }
      },

      res(res) {
        return {
          statusCode: res.statusCode
        }
      }
    }
  },

  namespace: "api",

  server: {
    name: "my-app",
    certificate: "/path/to/my.crt",
    key: "/path/to/my.key",
    log: Bunyan.createLogger(),
    middlewares: [
      restify.bodyParser(),
      restify.queryParser(),
      myCustomMiddleware()
    ]
  }
});

// wire up your routes
app.map(function () {
  this.resource("user");
  this.route("user/resetPassword", {
    using: "users:resetPassword", // controller:method
    method: "post" // request method
  });
});

app.start(3000).then(() => {
  console.log("App listening.")
});
```

The above will create the following route mapping

```
GET    /users               => UserController.index
GET    /users/:userId       => UserController.show
POST   /users               => UserController.create
PUT    /users/:userId       => UserController.update
DELETE /users/:userId       => UserController.destroy
POST   /users/resetPassword => UserController.resetPassword
```

### Router

The router handles route management and normalization, creating CRUD endpoints
for resources and normalizing all paths.

#### Resource

Use `resource` to generate a set of CRUD endpoints.

```javascript
app.map(function () {
  this.resource("user");
});

/**
 * GET    /users               => UserController.index
 * GET    /users/:userId       => UserController.show
 * POST   /users               => UserController.create
 * PUT    /users/:userId       => UserController.update
 * DELETE /users/:userId       => UserController.destroy
 */
```

#### Route

Use `route` to define a one off route.

```javascript
app.map(function () {
  this.route("/foos/bar", {
    using: "foo:getBar",
    method: "get"
  });
});

/**
 * GET /foos/bar => FooController.getBar
 */
```

#### Namespace

Use `namespace` to group a set of `routes` under a single base path. Namespace
takes an array of routes so follow the route api (with the addition of `path`)

```javascript
app.map(function () {
  this.namespace("users/:userId", [
    { path: "/account", using: "user:getAccount", method: "get" },
    { path: "/image", using: "user:setImage", method: "post" }
  ]);
});

/**
 * GET /users/:userId/account => UserController.getAccount
 * POST /users/:userId/image  => UserController.setImage
 */
```

### Controller

`lib/controllers/users.js`

```javascript
const parch = require("parch");

class UserController extends parch.Controller {
  constructor(options) {
    super(options);
  }

  index(req, res, next) {
    this.store.findAll(req.query).then(records => {
      /**
       * {
       *   users: [{
       *   ...
       *   }]
       * }
       */
       res.send(this.STATUS_CODES.SUCCESS, records);
    }).catch(next);
  }

  show(req, res, next) {
    this.store.findOne(req.params.id).then(record => {
      /**
       * {
       *   user: {
       *   ...
       *   }
       * }
       */
       res.send(this.STATUS_CODES.SUCCESS, record);
    }).catch(next);
  }

  create(req, res, next) {
    this.store.createRecord(req.body.user).then(record => {
      /**
       * {
       *   user: {
       *   ...
       *   }
       * }
       */
       res.send(this.STATUS_CODES.CREATED, record);
    }).catch(next);
  }

  update(req, res, next) {
    this.store.updateRecord(req.params.id, req.body).then(updatedRecord => {
      /**
       * {
       *   user: {
       *   ...
       *   }
       * }
       */
       res.send(this.STATUS_CODES.SUCCESS, record);
    }).catch(next);
  }

  destroy(req, res, next) {
    this.store.destroyRecord(req.params.id).then(() => {
      res.send(this.STATUS_CODES.NO_CONTENT);
    }).catch(next);
  }

  resetPassword(req, res, next) {
    this.store.findOne(req.params.id).then(record => {
      record.password = req.body.password;
      return record.save();
    }).then(record => {
      res.send(this.STATUS_CODES.SUCCESS);
    }).catch(next);
  }
}
```

### Controller Hooks [Deprecated]

**Controller hooks in this fashion are deprecated. Please use `beforeModel`, `model`, and `afterModel` structure instead. [see here for more](#controller-lifecycle)**

Controller hooks allow for pre and post processing of requests. Both before and
after hooks are supported as well as any additional methods added when using
`Controller#route` or `Controller#namespace`. When using the `after` hook, make
sure to call next after sending your response.

```javascript
class UserController extends parch.Controller {
  constructor(options) {
    super(options);

    this.hooks = {
      // The hook name must match the method
      index: {
        before(req, res, next) {
          return checkPermissions().then(() => {
            next();
          }).catch(next);
        },
        after(req, res, next) {
          req.log.child().info("post processing");
        }
      }
    };
  }

  index(req, res, next) {
    res.send(200);
    next();
  }
}
```

### Nested Controllers

Organizing your controllers into groups can make things more manageable. By storing
each action in a separate class, you will have tighter control over your logic.

```
├── lib
│   ├── controllers
│   │   ├── post
│   │   │   ├── create.js
│   │   │   ├── destroy.js
│   │   │   ├── index.js
│   │   │   ├── show.js
│   │   │   └── update.js
│   │   └── users
│   │       ├── create.js
│   │       ├── destroy.js
│   │       ├── index.js
│   │       ├── show.js
│   │       └── update.js
```

### Controller Lifecycle

Controller lifecycle can be handled by three main hooks. Each of these hooks take
the same arguments `(req, res, next)`

#### `beforeModel`

The `beforeModel` hook runs before the main model hook. In this method you have access to the request and response objects and it is assumed that the response
lifecycle has not been completed.

```javascript
export default class UserListController {
  beforeModel(req, res, next) {
    return utils.permissions(req)
      .then(() => next())
      .catch(next);
  }
}
```

#### `model`

The `model` hook is the main action of your controller class. This is where the
response object is assumed to end. **Note: if you also define an `afterModel` hook
you _must_ call next in the model hook.

```javascript
export default class UserListController {
  model(req, res, next) {
    this.store.findAll("user").then(users => {
      res.send(this.STATUS_CODES.SUCCESS, users);

      next();
    }).catch(next);
  }
}
```

#### `afterModel`

The `afterModel` is the last hook in the controller lifecycle. Here you can do
things like metrics or audit logs

```javascript
export default class UserListController {
  afterModel(req, res, next) {
    req.log.debug("afterModel hook for user list");
  }
}
```

### Model

Models are defined following the [sequelize define](http://docs.sequelizejs.com/manual/tutorial/models-definition.html)
pattern. [Options](http://docs.sequelizejs.com/manual/tutorial/models-definition.html) for the model definition can be passed to the constructor's `super` call

`lib/models/user.js`

```javascript
class UserModel extends parch.Model {
  constructor() {
    super({
      classMethods: {
      },

      getterMethods: {
      },

      hooks: {
      },

      instanceMethods: {
      }
    });
  }

  associate(User, models) {
    User.hasMany(models.Posts);
    User.hasMany(models.User, { as: "Parent" });
  }

  define(DataTypes) {
    const user = {
      email: {
        type: DataTypes.STRING,
        validate: { isEmail: true }
      }
    };

    return user;
  }
}
```

### Serializers

Serializers allow you to control how data is transformed and sent to the client.
Each model you define can also define an accompanying serializer to transform its
data. If you do not define one yourself, Parch will use the [JSONSerializer](https://github.com/dylanfoster/json-serializer)

JSONSerializer is the most basic of all the serializers, offering only a `normalizeResponse` method, which in turn just returns the instance or instance array.
To extend this behavior, the [RESTSerializer](https://github.com/dylanfoster/rest-serializer) will nest your record under a
singular or plural record key (e.g. `{ user: record  }` or `{ users: [records] }`), as well as automatically add each hasMany/belongsTo relationship as an array
of ids, removing the need for you to transform these yourself. The [JSONAPISerializer](https://github.com/dylanfoster/json-api-serializer)(TBD) takes this even
further, ensuring your data is transform following the JSONAPI spec

`lib/serializers/user.js`

```javascript
import { RestSerializer } from "parch";

export default UserSerializer extends RestSerializer {
  keyForRecord(record, singular) {}

  keyForRelationship(association) {}

  normalizeArrayResponse(instances, fallbackName) {}

  normalizeRelationships(instance) {}

  normalizeSingularResponse(instance) {}
}
```

## Authentication and Authorization

Authorization is handled using [jwt](https://jwt.io/), with more
options coming in the future. To disable auth for specific routes, use the
`authentication.unauthenticated` array. Empty by default, you can give a string
or regex expression to skip your unauthenticated routes

```javascript
const parch = new parch.Application({
  authentication: {
    unauthenticated: [/\/posts[\s\S]*/, "/users/resetPassword"]
  }
});
```

In order to authenticate a user, create and sign a JWT token to send back to the
client. The authorization middleware will then look for this token in the
`Authorization` header. [see jsonwebtoken](https://www.npmjs.com/package/jsonwebtoken)

```javascript
// lib/controllers/user_controller.js

const jwt = require("jsonwebtoken");

const config = require("../../config");

class UserController extends parch.Controller {
  constructor(settings) {
    super(settings);
  }

  authenticate(req, res, next) {
    this.model.findOne({ where: { email: req.body.email }}).then(user => {
      if (user) {
        const token = jwt.sign(user, config.secret);

        res.send(200, { token });
      } else {
        throw new this.errors.UnauthorizedError("email or password is invalid");
      }
    });
  }
}

module.exports = UserController;

...
// lib/app.js

app.map(function () {
  this.route("/users/authenticate", { using: "user:authenticate", method: "post" });
});
```

```bash
curl http://my-server.com/protectedRoute -H 'Authorization: Bearer <token>'
```

## Application Initializers

Initializers allow you to accomplish many things during application boot.
Registering mixins, add custom application logic, and adding services can all be
done in an initializer and attached to the application instance. Parch will run
your initializers in alphanumeric order. This means that if you need to run them
in a specific order you should prefix them in a way that will accomplish that.

`your-app/lib/initializers/my-awesome-initializer.js`

```javascript
"use strict";

const Worker = require("../worker");

module.exports = {
  initialize(appInstance, registry) {
    appInstance.foo = "bar";

    app.worker = new Worker();
  },

  name: "my-awesome-initializer"
};
```

## Logging

Logging is handled automatically for you. All requests and responses will be logged
using a custom [Bunyan instance](https://github.com/dylanfoster/parch/blob/master/src/logger.js).

## Error handling and responses

### Errors

Error handling is done using [restify-errors](https://github.com/restify/errors).
When using controller helpers (`findAll`, `findOne`, etc) errors are handled
automatically for you. Just catch your Promise with `next` and parch will handle
the rest.

```javascript
show(req, res, next) {
  this.findOne(req.params.id).then(user => {
  }).catch(next);
}

/**
 * { code: "NotFound", message: "User with id '1' does not exist" }
 */
```

Errors handled by parch:

 - `findOne`:
   - `NotFound`: The record does not exist
 - `createRecord`:
   - `BadRequest`: Request body was missing or invalid
   - `UnprocessableEntity`: Model validations failed
 - `updateRecord`:
   - `BadRequest`: Request body was missing or invalid
   - `UnprocessableEntity`: Model validations failed
 - `destroyRecord`:
   - `NotFound`: The record does not exist

Need to handle your own errors? `controller.errors` contains all of [restify-errors](https://github.com/restify/errors)' errors

### Responses

Parch also helps you standardize on your response statuses. Using [controller.STATUS_CODES](https://github.com/dylanfoster/parch/blob/develop/src/utils/status_codes.js)
you'll never have to worry about which status to send.

```javascript
show(req, res, next) {
  res.send(this.STATUS_CODES.SUCCESS) // 200
}
```

## Options

  - **authentication**
    - `secretKey(String)`: A secret string used to sign JWT tokens
    - `unauthenticated(Array)`: an array of strings or regex patterns to skip authentication.
  - **controllers**
    - `dir(String)`: The path to your controllers directory. **Default**: `__dirname/controllers`
  - **database**
    - `connection(Object)` [Sequelize connection options](http://docs.sequelizejs.com/en/latest/docs/getting-started/)
    - `models`
      - `dir(String)`: The path to your models directory. **Default**: `__dirname/models`
  - **initializers**
    - `dir(String)`: The path to your initializers directory. **Default**: `__dirname/initializers`
  - **logging**
    - `dir(String)`: Path where logs should be saved
    - `logger(Object)` **optional**: An optional logger instance
    - `serializers(Object)`:
      - `req(Function)`: your request serializer. takes the request as its only argument
      - `res(Function)`: your response serializer. takes the response as its only argument
  - **namespace**: Set the base namespace for all routes and resources (e.g. `api`)
  - **serializers**
    - `dir(String)`: The path to your serializers directory. **Default*: `__dirname/serializers`
  - **server** All options (*with the exception of `middlewares`*) are passed directly to [restify](http://restify.com/#creating-a-server)
    - `log`: defaults to parch's [bunyan instance](https://github.com/dylanfoster/parch/blob/master/src/logger.js) but can be overridden
    - `middlewares(Array)`: merged with parch's [default middlwares](https://github.com/dylanfoster/parch/blob/master/src/application.js#L24-L31)


text:
{-# LANGUAGE ScopedTypeVariables #-}
module Database.Persist.TX.Utils where

-- {{{1
import ClassyPrelude                        hiding (delete)
#if !MIN_VERSION_classy_prelude(1, 5, 0)
import Control.DeepSeq                      (NFData(..), deepseq)
#endif
import Data.Proxy (Proxy(..))
import Data.Aeson                           (Value, ToJSON(..), FromJSON(..))
import Data.ByteString.Builder              (toLazyByteString)
import Data.Char
import Data.Conduit
import Data.List (dropWhileEnd)
import Data.Time
import Database.Persist
import Database.Persist.Sql
#if MIN_VERSION_persistent(2, 12, 0)
import Database.Persist.SqlBackend.Internal
#endif

import Database.PostgreSQL.Simple.Time
import System.FilePath

import Control.Monad.Except                 (ExceptT, MonadError(..))

import qualified Data.Aeson                 as A
import qualified Data.List                  as L

import Control.Monad.State.Strict           (StateT)
import qualified Control.Monad.State.Strict as S

import Language.Haskell.TH
-- }}}1

type SqlWriteSource m a = forall backend. SqlBackendCanWrite backend =>
#if MIN_VERSION_conduit(1, 3, 0)
  ConduitT () a (ReaderT backend m) ()
#else
  Source (ReaderT backend m) a
#endif

type SqlWriteConduit i m o = forall backend. SqlBackendCanWrite backend =>
#if MIN_VERSION_conduit(1, 3, 0)
  ConduitT i o (ReaderT backend m) ()
#else
  Conduit i (ReaderT backend m) o
#endif

type SqlWriteSink i m r = forall backend. SqlBackendCanWrite backend =>
#if MIN_VERSION_conduit(1, 3, 0)
  ConduitT i Void (ReaderT backend m) r
#else
  Sink i (ReaderT backend m) r
#endif

type SqlReadSource m a = forall backend. SqlBackendCanRead backend =>
#if MIN_VERSION_conduit(1, 3, 0)
  ConduitT () a (ReaderT backend m) ()
#else
  Source (ReaderT backend m) a
#endif

type SqlReadConduit i m o = forall backend. SqlBackendCanRead backend =>
#if MIN_VERSION_conduit(1, 3, 0)
  ConduitT i o (ReaderT backend m) ()
#else
  Conduit i (ReaderT backend m) o
#endif

type SqlReadSink i m r = forall backend. SqlBackendCanRead backend =>
#if MIN_VERSION_conduit(1, 3, 0)
  ConduitT i Void (ReaderT backend m) r
#else
  Sink i (ReaderT backend m) r
#endif

#if MIN_VERSION_persistent(2, 8, 0)
type DBMonadBase m = (MonadUnliftIO m)
#else
type DBMonadBase m = (MonadBaseControl IO m)
#endif

class DBActionRunner a where
    -- | This should be a monad transformer
    type DBAction a :: (* -> *) -> * -> *
    runDBWith :: DBMonadBase m => a -> DBAction a m r -> m r

-- | Example: SomeDBActionRunner SqlPersistT
data SomeDBActionRunner t = forall a. (DBActionRunner a, DBAction a ~ t)
                            => SomeDBActionRunner a

runDBWith' :: DBMonadBase m
           => SomeDBActionRunner t
           -> t m r
           -> m r
runDBWith' (SomeDBActionRunner x) = runDBWith x

-- | A little "generalized" version of 'liftPersist'
runPersistEnvReaderT :: ( MonadReader env m
#if MIN_VERSION_persistent(2, 6, 0)
                        , HasPersistBackend env, BaseBackend env ~ r
#else
                        , HasPersistBackend env r
#endif
                        )
                     => ReaderT r m b -> m b
runPersistEnvReaderT f = do
  e <- ask
  runReaderT f (persistBackend e)

rnfEntity ::
    ( NFData a
#if MIN_VERSION_persistent(2, 0, 0)
    , NFData (Key a)
#else
    , NFData (KeyBackend (PersistEntityBackend a) a)
#endif
    ) =>
    Entity a -> ()
rnfEntity (Entity k v) = k `deepseq` v `deepseq` ()

type PersistQueryUniqueMonad backend n m =
#if MIN_VERSION_persistent(2, 0, 0)
    ( PersistUnique backend
    , PersistQuery backend
        -- PersistUnique/PersistQuery implies PersistStore
    , m ~ ReaderT backend n
    , MonadIO n
    )
#else
    ( PersistUnique m
    , PersistQuery m
        -- PersistUnique/PersistQuery implies PersistStore
    , backend ~ ()
    , n ~ []
    )
#endif

type IsPersistMonadOf backend n m val =
#if MIN_VERSION_persistent(2, 5, 0)
    ( PersistRecordBackend val backend
    , m ~ ReaderT backend n
    )
#elif MIN_VERSION_persistent(2, 0, 0)
    ( PersistEntityBackend val ~ backend
    , PersistEntity val
    , m ~ ReaderT backend n
    )
#else
    ( PersistEntityBackend val ~ PersistMonadBackend m
    , PersistEntity val
    , n ~ []
    , backend ~ ()
    )
#endif

type PersistUniqueMonad backend n m =
#if MIN_VERSION_persistent(2, 0, 0)
    ( PersistUnique backend
        -- PersistUnique implies PersistStore
    , m ~ ReaderT backend n
    , MonadIO n
    )
#else
    ( PersistUnique m
        -- PersistUnique implies PersistStore
    , backend ~ ()
    , n ~ []
    )
#endif

type PersistQueryMonad backend n m =
#if MIN_VERSION_persistent(2, 0, 0)
    ( PersistQuery backend
        -- PersistQuery implies PersistStore
    , m ~ ReaderT backend n
    , MonadIO n
    )
#else
    ( PersistQuery m
        -- PersistQuery implies PersistStore
    , backend ~ ()
    , n ~ []
    )
#endif

type PersistStoreMonad backend n m =
#if MIN_VERSION_persistent(2, 0, 0)
    ( PersistStore backend
    , m ~ ReaderT backend n
    , MonadIO n
    )
#else
    ( PersistStore m
    , backend ~ ()
    , n ~ []
    )
#endif

-- | try to insert a new record, replace existing one if unique constaint conflicts.
insertOrReplace' :: ( PersistQueryUniqueMonad backend n m
                    , IsPersistMonadOf backend n m val
#if MIN_VERSION_persistent(2, 10, 0)
                    , AtLeastOneUniqueKey val
#endif
                    )
                 => val
                 -> m (Either (Key val) (Key val))
                 -- ^ Left: the old key; Right: the new key
insertOrReplace' v = insertBy v
                        >>= either
                                (\(Entity k _) -> replace k v >> return (Left k))
                                (return . Right)

insertOrReplace :: ( PersistQueryUniqueMonad backend n m
                    , IsPersistMonadOf backend n m val
#if MIN_VERSION_persistent(2, 10, 0)
                    , AtLeastOneUniqueKey val
#endif
                    )
                 => val
                 -> m (Key val)
insertOrReplace = fmap (either id id) . insertOrReplace'

-- | select current records, replace them with the supplied new list.
-- Try hard to retain old records that are the same as new ones.
insertOrUpdateWithList ::
    ( Eq val
    , PersistQueryUniqueMonad backend n m
    , IsPersistMonadOf backend n m val
#if MIN_VERSION_persistent(2, 10, 0)
    , AtLeastOneUniqueKey val
#endif
    ) =>
    [Filter val]
    -> [val]                -- ^ new values
    -> m ([Key val], [Key val])
                            -- ^ new created keys and keys that are not touched
insertOrUpdateWithList fts new_ones = do
    old_entities <- selectList fts []
    let to_be_deleted = catMaybes $ flip map old_entities $ \(Entity k v) ->
                            if v `elem` new_ones
                                then Nothing
                                else Just k
    let to_be_inserted = catMaybes $ flip map new_ones $ \v ->
                            if v `elem` map entityVal old_entities
                                then Nothing
                                else Just v
    new_keys <- forM to_be_inserted $ \v -> do
                    insertOrReplace v
    return $ (new_keys, to_be_deleted L.\\ new_keys)

-- | like insertOrUpdateWithList, but also delete untouched keys.
replaceWithList ::
    ( Eq val
#if MIN_VERSION_persistent(2, 0, 0)

#if MIN_VERSION_persistent(2, 5, 0)
    , PersistRecordBackend val backend
#else
    , PersistEntityBackend val ~ backend
    , PersistEntity val
#endif

    , PersistUnique backend
    , PersistQuery backend
    , m ~ ReaderT backend n
    , MonadIO n

#else
    , PersistEntityBackend val ~ PersistMonadBackend m
    , PersistUnique m
    , PersistQuery m
#endif
#if MIN_VERSION_persistent(2, 10, 0)
    , AtLeastOneUniqueKey val
#endif
    ) =>
    [Filter val]
    -> [val]                -- ^ new values
    -> m ([Key val], [Key val])
replaceWithList fts new_ones = do
    (new_keys, to_be_deleted) <- insertOrUpdateWithList fts new_ones
    mapM_ delete to_be_deleted
    return (new_keys, to_be_deleted)

-- | Automatically undo transaction when there is error throwError'ed
-- in wrapped function.
undoTransWhenE ::
#if MIN_VERSION_persistent(2, 0, 0)
    (MonadReader SqlBackend m, MonadIO m, MonadError e m) =>
    m a -> m a
#else
    (MonadSqlPersist m, MonadError e m) => m a -> m a
#endif
undoTransWhenE f = catchError f h
    where
        h err = liftPersist transactionUndo >> throwError err

#if MIN_VERSION_persistent(2, 0, 0)
undoTransWhenE2 :: (MonadIO m, MonadError e m) =>
                ReaderT SqlBackend m a
                -> ReaderT SqlBackend m a
undoTransWhenE2 f = catchError f h
    where
        h err = transactionUndo >> throwError err
#endif

-- | wrapped 'get', throwError when record does not exist.
getOrE ::
    ( PersistStore backend
#if MIN_VERSION_persistent(2, 0, 0)

#if MIN_VERSION_persistent(2, 5, 0)
    , PersistRecordBackend val backend
#else
    , PersistEntityBackend val ~ backend
    , PersistEntity val
#endif

    , m ~ ReaderT backend n
    , MonadIO n
#else
    , PersistStore m
    , PersistEntityBackend val ~ PersistMonadBackend m
#endif
    ) =>
    e -> Key val -> ExceptT e m val
getOrE err k = lift (get k) >>= maybe (throwError err) return

#if MIN_VERSION_persistent(2, 0, 0)
getOrE2 :: ( PersistStore backend

#if MIN_VERSION_persistent(2, 5, 0)
            , PersistRecordBackend val backend
#else
            , PersistEntityBackend val ~ backend
            , PersistEntity val
#endif

            , MonadIO m
            , MonadError e m
            )
        => e
        -> Key val
        -> ReaderT backend m val
getOrE2 err k = get k >>= maybe (throwError err) return
#endif

-- | wrapped 'getBy', throwError when record does not exist.
getByOrE ::
    ( PersistUnique backend
#if MIN_VERSION_persistent(2, 0, 0)

#if MIN_VERSION_persistent(2, 5, 0)
    , PersistRecordBackend val backend
#else
    , PersistEntityBackend val ~ backend
    ,  PersistEntity val
#endif

    , m ~ ReaderT backend n
    , MonadIO n
#else
    , PersistEntityBackend val ~ PersistMonadBackend m
    , PersistUnique m
#endif
    ) =>
    e -> Unique val -> ExceptT e m (Entity val)
getByOrE err k = lift (getBy k) >>= maybe (throwError err) return

#if MIN_VERSION_persistent(2, 0, 0)
getByOrE2 ::
    ( PersistUnique backend
#if MIN_VERSION_persistent(2, 5, 0)
    , PersistRecordBackend val backend
#else
    , PersistEntityBackend val ~ backend
    , PersistEntity val
#endif

    , MonadIO m, MonadError e m
    ) =>
    e -> Unique val -> ReaderT backend m (Entity val)
getByOrE2 err k = getBy k >>= maybe (throwError err) return
#endif

-- | update or replace a record
insertOrUpdate ::
    ( PersistUnique backend
#if MIN_VERSION_persistent(2, 0, 0)

#if MIN_VERSION_persistent(2, 5, 0)
    , PersistRecordBackend val backend
#else
    , PersistEntityBackend val ~ backend
    , PersistEntity val
#endif

    , m ~ ReaderT backend n
    , MonadIO n
#else
    , PersistEntityBackend val ~ PersistMonadBackend m
    , PersistUnique m, PersistQuery m
#endif

#if MIN_VERSION_persistent(2, 10, 0)
    , AtLeastOneUniqueKey val
#endif
    ) =>
    val                 -- ^ new value to insert
    -> [Update val]     -- ^ update commands to exec when old record
                        -- with the same unique key already exists.
    -> m (Either (Key val) (Key val))
                        -- ^ Left : the new created id
                        -- ^ Right: the id of old record that has been updated
insertOrUpdate v updates = do
    insertBy v >>= either (upd . entityKey) (return . Left)
    where
        upd k = update k updates >> return (Right k)

-- | Delete all records matching the given criterion
-- , except those whose keys in a specific list.
deleteWhereExcept ::
    ( PersistQuery backend
#if MIN_VERSION_persistent(2, 0, 0)

#if MIN_VERSION_persistent(2, 5, 0)
    , PersistRecordBackend val backend
#else
    , PersistEntityBackend val ~ backend
    , PersistEntity val
#endif

    , m ~ ReaderT backend n
    , MonadIO n
#else
    , PersistEntityBackend val ~ PersistMonadBackend m
    , PersistStore m, PersistQuery m
#endif
    ) =>
    [Filter val]
    -> [Key val]
    -> m [Key val]
deleteWhereExcept filters keeps = do
    ks <- selectKeysList filters []
    let to_del = ks L.\\ keeps
    mapM_ delete to_del
    return to_del

type CachedInMap val m = StateT (Map (Key val) val) m

-- | when we don't should that might reselect the same record multiple times,
-- use this function to reduce DB access.
cget ::
    ( PersistStore backend
#if MIN_VERSION_persistent(2, 0, 0)

#if MIN_VERSION_persistent(2, 5, 0)
    , PersistRecordBackend val backend
#else
    , PersistEntityBackend val ~ backend
    , PersistEntity val
#endif

    , m ~ ReaderT backend n
    , MonadIO n
#else
    , PersistEntityBackend val ~ PersistMonadBackend m
    , PersistStore m
#endif
    )
    => Key val
    -> CachedInMap val m (Maybe val)
cget k = do
    mv <- S.gets $ lookup k
    case mv of
        Just v -> return $ Just v
        Nothing -> do
            mv2 <- lift $ get k
            maybe (return ()) (S.modify . insertMap k) mv2
            return mv2

cselectList ::
    ( PersistQuery backend
#if MIN_VERSION_persistent(2, 0, 0)

#if MIN_VERSION_persistent(2, 5, 0)
    , PersistRecordBackend val backend
#else
    , PersistEntityBackend val ~ backend
    , PersistEntity val
#endif

    , m ~ ReaderT backend n
    , MonadIO n
#else
    , PersistEntityBackend val ~ PersistMonadBackend m
    , PersistQuery m
#endif
    )
    => [Filter val]
    -> [SelectOpt val]
    -> CachedInMap val m [Entity val]
cselectList f o = do
    records <- lift $ selectList f o
    mapM_ cput records
    return records

cput ::
    (
#if MIN_VERSION_persistent(2, 0, 0)

#if MIN_VERSION_persistent(2, 5, 0)
    PersistRecordBackend val backend
#else
    PersistEntityBackend val ~ backend
    , PersistEntity val
#endif

    , m ~ ReaderT backend n
    , MonadIO n
#else
    , PersistEntityBackend val ~ PersistMonadBackend m
    , PersistStore m
#endif
    ) =>
    Entity val -> CachedInMap val m ()
cput (Entity k v) = S.modify $ insertMap k v

getFieldDBName :: PersistEntity a
               => EntityField a typ
#if MIN_VERSION_persistent(2, 12, 0)
               -> FieldNameDB
#else
               -> DBName
#endif
getFieldDBName = fieldDB . persistFieldDef

escFieldDBName :: PersistEntity a =>
#if MIN_VERSION_persistent(2, 0, 0)
    SqlBackend
#else
    Connection
#endif
    -> EntityField a typ -> Text
escFieldDBName conn =
#if MIN_VERSION_persistent(2, 12, 0)
  connEscapeFieldName conn
#else
  connEscapeName conn
#endif
    . getFieldDBName

escEntityDBName :: (PersistEntity a, Monad m) =>
#if MIN_VERSION_persistent(2, 0, 0)
    SqlBackend
#else
    Connection
#endif
    -> m a -> Text
escEntityDBName conn =
#if MIN_VERSION_persistent(2, 12, 0)
  connEscapeTableName conn
#else
  connEscapeName conn . entityDB
#endif
   . entityDef

sinkEntityAsMap :: (Monad m, Ord (Key a))
#if MIN_VERSION_conduit(1, 3, 0)
                => ConduitT (Entity a) Void m (Map (Key a) a)
#else
                => Sink (Entity a) m (Map (Key a) a)
#endif
sinkEntityAsMap = go mempty
    where
        go s = do
            mx <- await
            case mx of
                Nothing             -> return s
                Just (Entity k v)   -> go $ insertMap k v s

-- | escape char that is considered as "special" in string with specified escape char
addEscape :: [Char] -> Char -> String -> String
addEscape   _           _   []      = []
addEscape   specials    esc (x:xs)  =
    let next = x : addEscape specials esc xs
    in if x == esc || x `elem` specials
        then esc : next
        else next

removeEscape :: [Char] -> Char -> String -> String
removeEscape    _        _   []          = []
removeEscape    _        _   [x]         = [x]
removeEscape    specials esc (x1:x2:xs)  =
    if x1 == esc
        then if x2 == esc || x2 `elem` specials
                then x2 : f xs
                else x2 : f xs  -- as tradition
        else x1 : f (x2:xs)
    where
        f = removeEscape specials esc

-- | Sepcial chars used in 'LIKE' pattern in SQL
-- XXX: This does not work for all DBMS
--      MS SQL Server has more special chars than others.
unsafeSpecialsForSqlLike :: [Char]
unsafeSpecialsForSqlLike = "%_"

unsafeEscapeForSqlLike :: String -> String
unsafeEscapeForSqlLike =
    -- XXX: 看 persistent 的代码,目前无法加上 ESCAPE 'x' 语句,
    -- 另一方面,不是所有DBMS都有缺省的转义字符(见Sqlite, Oracle, MS SQL Server)
    -- 还好,PostgreSQL MySQL 都有相同的缺省转义字符 '\'
    addEscape unsafeSpecialsForSqlLike '\\'

unsafeEscapeForSqlLikeT :: Text -> Text
unsafeEscapeForSqlLikeT = pack . unsafeEscapeForSqlLike . unpack

unsafeRemoveEscapeForSqlLike :: String -> String
unsafeRemoveEscapeForSqlLike = removeEscape unsafeSpecialsForSqlLike '\\'

unsafeRemoveEscapeForSqlLikeT :: Text -> Text
unsafeRemoveEscapeForSqlLikeT = pack . unsafeRemoveEscapeForSqlLike . unpack

contains :: EntityField v Text -> Text -> Filter v
contains = contains2 id

contains2 :: (PersistField a) =>
    (Text -> a)
    -> EntityField v a -> Text -> Filter v
contains2 conv field val = Filter field
#if MIN_VERSION_persistent(2, 10, 0)
                        (FilterValue v)
#else
                        (Left v)
#endif
                        (BackendSpecificFilter " LIKE ")
  where v = conv $ mconcat ["%", unsafeEscapeForSqlLikeT val, "%"]

-- | 包含并忽略大小写
containsI :: EntityField v Text -> Text -> Filter v
containsI = containsI2 id

-- | 包含并忽略大小写
containsI2 :: (PersistField a)
           => (Text -> a)
           -> EntityField v a
           -> Text
           -> Filter v
containsI2 conv field val = Filter field
#if MIN_VERSION_persistent(2, 10, 0)
                        (FilterValue v)
#else
                        (Left v)
#endif
                        (BackendSpecificFilter " ILIKE ")
  where v = conv $ mconcat ["%", unsafeEscapeForSqlLikeT val, "%"]

-- | Store data as native 'json' data type of DB engine
-- supported DB engines: PostgreSQL, MySQL
newtype PersistJson = PersistJson { unPersistJson :: Value }
  deriving (Show, Eq)

instance PersistFieldSql PersistJson where
  sqlType _ = SqlOther "JSON"

instance PersistField PersistJson where
#if MIN_VERSION_persistent(2, 11, 0)
  toPersistValue = PersistLiteralEscaped . toStrict . A.encode . unPersistJson
#else
  toPersistValue = PersistDbSpecific . toStrict . A.encode . unPersistJson
#endif

#if MIN_VERSION_persistent(2, 11, 0)
  fromPersistValue (PersistLiteralEscaped bs) =
                                            case A.eitherDecode' $ fromStrict bs of
                                              Left err -> Left $ fromString err
                                              Right x  -> Right $ PersistJson x
  fromPersistValue (PersistLiteral bs) = case A.eitherDecode' $ fromStrict bs of
                                           Left err -> Left $ fromString err
                                           Right x  -> Right $ PersistJson x
#else
  fromPersistValue (PersistDbSpecific bs) = case A.eitherDecode' $ fromStrict bs of
                                              Left err -> Left $ fromString err
                                              Right x  -> Right $ PersistJson x
#endif

  fromPersistValue (PersistByteString bs) = case A.eitherDecode' $ fromStrict bs of
                                              Left err -> Left $ fromString err
                                              Right x  -> Right $ PersistJson x

  fromPersistValue x = Left $ "PersistJson must be converted from PersistDbSpecific, but got " <> tshow x

instance ToJSON PersistJson where toJSON = unPersistJson

instance FromJSON PersistJson where parseJSON = return . PersistJson

toPersistJson :: ToJSON a => a -> PersistJson
toPersistJson = PersistJson . toJSON

fromPersistJson :: FromJSON a => PersistJson -> Either String a
fromPersistJson (PersistJson v) = case A.fromJSON v of
                                    A.Error err -> Left err
                                    A.Success x -> Right x

-- | Store data as native 'jsonb' data type of postgresql
newtype PersistJsonb = PersistJsonb { unPersistJsonb :: Value }
  deriving (Show, Eq)

instance PersistFieldSql PersistJsonb where
  sqlType _ = SqlOther "JSONB"

instance PersistField PersistJsonb where
#if MIN_VERSION_persistent(2, 11, 0)
  toPersistValue = PersistLiteralEscaped . toStrict . A.encode . unPersistJsonb
#else
  toPersistValue = PersistDbSpecific . toStrict . A.encode . unPersistJsonb
#endif

#if MIN_VERSION_persistent(2, 11, 0)
  fromPersistValue (PersistLiteralEscaped bs) = case A.eitherDecode' $ fromStrict bs of
                                                  Left err -> Left $ fromString err
                                                  Right x  -> Right $ PersistJsonb x
  fromPersistValue (PersistLiteral bs) = case A.eitherDecode' $ fromStrict bs of
                                           Left err -> Left $ fromString err
                                           Right x  -> Right $ PersistJsonb x
#else
  fromPersistValue (PersistDbSpecific bs) = case A.eitherDecode' $ fromStrict bs of
                                              Left err -> Left $ fromString err
                                              Right x  -> Right $ PersistJsonb x
#endif

  fromPersistValue (PersistByteString bs) = case A.eitherDecode' $ fromStrict bs of
                                              Left err -> Left $ fromString err
                                              Right x  -> Right $ PersistJsonb x

  fromPersistValue x = Left $ "PersistJsonb must be converted from PersistDbSpecific, but got " <> tshow x

instance ToJSON PersistJsonb where toJSON = unPersistJsonb

instance FromJSON PersistJsonb where parseJSON = return . PersistJsonb

toPersistJsonb :: ToJSON a => a -> PersistJsonb
toPersistJsonb = PersistJsonb . toJSON

fromPersistJsonb :: FromJSON a => PersistJsonb -> Either String a
fromPersistJsonb (PersistJsonb v) = case A.fromJSON v of
                                    A.Error err -> Left err
                                    A.Success x -> Right x

newtype PersistDiffTime = PersistDiffTime { unPersistDiffTime :: NominalDiffTime }
  deriving (Show, Eq, Num)

instance PersistFieldSql PersistDiffTime where
  sqlType _ = SqlOther "INTERVAL"

instance PersistField PersistDiffTime where
#if MIN_VERSION_persistent(2, 11, 0)
  toPersistValue = PersistLiteralEscaped . toStrict . toLazyByteString . nominalDiffTimeToBuilder . unPersistDiffTime
  fromPersistValue (PersistLiteralEscaped bs) = Left $ "TBD: " <> decodeUtf8 bs -- TODO: decode interval from different styles output
  fromPersistValue (PersistLiteral bs) = Left $ "TBD: " <> decodeUtf8 bs -- TODO: decode interval from different styles output
#else
  toPersistValue = PersistDbSpecific . toStrict . toLazyByteString . nominalDiffTimeToBuilder . unPersistDiffTime
  fromPersistValue (PersistDbSpecific bs) = Left $ "TBD: " <> decodeUtf8 bs -- TODO: decode interval from different styles output
#endif
  fromPersistValue x = Left $ "PersistDiffTime must be converted from PersistDbSpecific, but got " <> tshow x

findByEntityKey :: (Element seq ~ Entity record, SemiSequence seq, Eq (Key record))
                => Key record
                -> seq
                -> Maybe (Entity record)
findByEntityKey k = find ((== k) . entityKey)

entitiesToMap :: (ContainerKey c ~ Key record, MapValue c ~ record, IsMap c)
              => [Entity record]
              -> c
entitiesToMap = mapFromList . map (entityKey &&& entityVal)

-- | 字段名列表
entityFieldHaskellNames :: forall record. (PersistEntity record)
                        => Proxy record
#if MIN_VERSION_persistent(2, 12, 0)
                        -> [ FieldNameHS ]
#else
                        -> [ HaskellName ]
#endif
entityFieldHaskellNames _p =
#if MIN_VERSION_persistent(2, 12, 0)
  map fieldHaskell $
    getEntityFields $
      entityDef _p
#else
  map fieldHaskell $
    entityFields $
      entityDef (Nothing :: Maybe record)
#endif

compareByEntityFieldHaskellNameOrder :: (PersistEntity record) => Proxy record -> Text -> Text -> Ordering
compareByEntityFieldHaskellNameOrder p x y =
  case (m_px, m_py) of
    (Just px, Just py) -> compare px py
    _ -> compare x y
  where
    names = "id" :
              ( map
#if MIN_VERSION_persistent(2, 12, 0)
                unFieldNameHS
#else
                unHaskellName
#endif
                (entityFieldHaskellNames p)
              )
    m_px = L.elemIndex x names
    m_py = L.elemIndex y names

-- | 例如可以把 'get' 的返回值变成 Maybe (Entity a)
attachEntityKey :: (PersistEntity record, Functor m, Functor f)
                => (Key record -> m (f record))
                -> Key record
                -> m (f (Entity record))
attachEntityKey = attachEntityKey' Entity

attachEntityKey' :: (PersistEntity record, Functor m, Functor f)
                 => (Key record -> record -> a)
                 -> (Key record -> m (f record))
                 -> Key record
                 -> m (f a)
attachEntityKey' h g rec_id = fmap (fmap (h rec_id)) $ g rec_id

-- | 含有 deleted 字段的 PersistEntity
class PersistEntity a => HasEntityFieldDeleted a where
  entityFieldDeleted :: EntityField a Bool

deriveHasEntityFieldDeleted :: Name -> DecsQ
deriveHasEntityFieldDeleted n = do
  let deleted_field = mkName $ nameBase n <> "Deleted"
  return
    [ InstanceD Nothing [] (ConT ''HasEntityFieldDeleted `AppT` ConT n)
      [ FunD 'entityFieldDeleted
        [ Clause [] (NormalB $ ConE deleted_field) []
        ]
      ]
    ]

selectListWithDeleted :: ( HasEntityFieldDeleted a, PersistQueryMonad backend n m
                         , IsPersistMonadOf backend n m a
                         )
                      => Bool
                      -> [ Filter a ]
                      -> [ SelectOpt a ]
                      -> m [Entity a]
selectListWithDeleted is_deleted filters opts =
  selectList ((entityFieldDeleted ==. is_deleted) : filters) opts

selectKeysListWithDeleted :: ( HasEntityFieldDeleted a, PersistQueryMonad backend n m
                             , IsPersistMonadOf backend n m a
                             )
                          => Bool
                          -> [ Filter a ]
                          -> [ SelectOpt a ]
                          -> m [Key a]
selectKeysListWithDeleted is_deleted filters opts =
  selectKeysList ((entityFieldDeleted ==. is_deleted) : filters) opts

-- | insert record, retry when new record when insertUniqueEntity failed.
-- CAUTION: 若 a 包含多个 Unique a, 可能有死循环.请用 checkUniqueInsertRetry
insertUniqueEntityRetry :: ( PersistUniqueMonad backend n m
                           , IsPersistMonadOf backend n m a
                           )
                        => (a -> m (Maybe a))
                        -> a
                        -> m (Maybe (Entity a))
insertUniqueEntityRetry mk_new rec0 = loop rec0
  where
    loop rec = do
      m_ent <- insertUniqueEntity rec
      case m_ent of
        Just x -> return $ Just x
        Nothing -> mk_new rec >>= fmap join . mapM loop

-- | 效果类似于 insertUniqueEntityRetry,不过能处理有多个unique存在的情况
checkUniqueInsertRetry :: ( PersistUniqueMonad backend n m
                          , IsPersistMonadOf backend n m a
                          , Eq (Unique a)
                          )
                       => (a -> Unique a)
                       -> (a -> m (Maybe a))
                       -> a
                       -> m (Either (Unique a) (Maybe (Entity a)))
checkUniqueInsertRetry get_unique mk_new rec0 = loop rec0
  where
    loop rec = do
      m_unique <- checkUnique rec
      case m_unique of
        Nothing -> fmap (Right . Just) $ insertEntity rec
        Just u -> if u == get_unique rec
                     then mk_new rec >>= fmap (fromMaybe $ Right Nothing) . mapM loop
                     else return $ Left u

-- | 常见的自动改文件名策略: 在旧文件名后加上 (n) 的字串,n根据前一个文件名中的数字增加一
autoNameNumAscending :: FilePath -> FilePath
autoNameNumAscending p = replaceBaseName p new_base_name
  where base_name = takeBaseName p

        (file_real_name, file_seq_num) = fromMaybe (base_name, 0) split_num_part
        new_base_name = file_real_name <> "(" <> show (file_seq_num + 1) <> ")"

        split_num_part :: Maybe (FilePath, Integer)
        split_num_part = do
          s1 <- stripSuffix ")" base_name
          let s2 = dropWhileEnd isDigit s1
          real_name <- stripSuffix "(" s2
          guard $ not $ null real_name
          num_part <- stripPrefix s2 s1
          num <- readMay num_part
          guard $ num > 0 && num < 9999
          pure (real_name, num)

-- vim: set foldmethod=marker:


text:
set -e
./configure --prefix=$PREFIX --disable-gtk-doc || { cat config.log ; exit 1 ; }
make
make install

cd $PREFIX
rm -rf share/gtk-doc


text:
RSpec.describe Telegram::Bot::UpdatesController do
  include_context 'telegram/bot/updates_controller'
  let(:params) { {arg: 1, 'other_arg' => 2} }
  let(:respond_type) { :photo }
  let(:result) { double(:result) }
  let(:payload_type) { :message }
  let(:payload) { {message_id: double(:message_id), chat: chat} }
  let(:chat) { {id: double(:chat_id)} }

  shared_examples 'missing chat' do
    context 'when chat is missing' do
      let(:payload_type) { :some_type }
      it { expect { subject }.to raise_error(/chat/) }
    end
  end

  describe '#respond_with' do
    subject { controller.respond_with respond_type, params }
    include_examples 'missing chat'

    it 'sets chat_id & reply_to_message_id' do
      expect(bot).to receive("send_#{respond_type}").
        with(params.merge(chat_id: chat[:id])) { result }
      should eq result
    end
  end

  describe '#reply_with' do
    subject { controller.reply_with respond_type, params }
    include_examples 'missing chat'

    it 'sets chat_id & reply_to_message_id' do
      expect(bot).to receive("send_#{respond_type}").with(params.merge(
        chat_id: chat[:id],
        reply_to_message_id: payload[:message_id],
      )) { result }
      should eq result
    end

    context 'when update is not set' do
      let(:controller_args) { [bot, chat: deep_stringify(chat)] }
      it 'sets chat_id' do
        expect(bot).to receive("send_#{respond_type}").
          with(params.merge(chat_id: chat[:id])) { result }
        should eq result
      end
    end
  end

  describe '#edit_message' do
    subject { controller.edit_message(type, params) }
    let(:type) { :reply_markup }

    it { expect { subject }.to raise_error(/Can not edit message without/) }

    context 'when inline_message_id is present' do
      let(:payload) { {inline_message_id: double(:message_id)} }
      it 'sets inline_message_id' do
        expect(bot).to receive("edit_message_#{type}").with(params.merge(
          inline_message_id: payload[:inline_message_id],
        )) { result }
        should eq result
      end
    end

    context 'when message is present' do
      let(:payload) { {message: super().merge(chat: chat)} }
      it 'sets chat_id & message_id' do
        expect(bot).to receive("edit_message_#{type}").with(params.merge(
          message_id: payload[:message][:message_id],
          chat_id: payload[:message][:chat][:id],
        )) { result }
        should eq result
      end
    end
  end
end


text:
"""Remove data file Received columns from genomic_gc_validation_metrics

Revision ID: 42428d88dd1d
Revises: f0d635507938, 2ddb58c2b603, ace982c2cb2b, 3bd1cf3a498d
Create Date: 2022-09-16 09:10:07.627665

"""
from alembic import op
import sqlalchemy as sa
from sqlalchemy.dialects import mysql

# revision identifiers, used by Alembic.
revision = '42428d88dd1d'
down_revision = ('f0d635507938', '2ddb58c2b603', 'ace982c2cb2b', '3bd1cf3a498d')
branch_labels = None
depends_on = None

def upgrade(engine_name):
    globals()["upgrade_%s" % engine_name]()

def downgrade(engine_name):
    globals()["downgrade_%s" % engine_name]()

def upgrade_rdr():
    # ### commands auto generated by Alembic - please adjust! ###
    op.drop_column('genomic_gc_validation_metrics', 'vcf_md5_received')
    op.drop_column('genomic_gc_validation_metrics', 'raw_vcf_tbi_received')
    op.drop_column('genomic_gc_validation_metrics', 'raw_vcf_md5_received')
    op.drop_column('genomic_gc_validation_metrics', 'idat_red_received')
    op.drop_column('genomic_gc_validation_metrics', 'vcf_received')
    op.drop_column('genomic_gc_validation_metrics', 'crai_received')
    op.drop_column('genomic_gc_validation_metrics', 'cram_received')
    op.drop_column('genomic_gc_validation_metrics', 'gvcf_received')
    op.drop_column('genomic_gc_validation_metrics', 'gvcf_md5_received')
    op.drop_column('genomic_gc_validation_metrics', 'hf_vcf_md5_received')
    op.drop_column('genomic_gc_validation_metrics', 'idat_green_md5_received')
    op.drop_column('genomic_gc_validation_metrics', 'cram_md5_received')
    op.drop_column('genomic_gc_validation_metrics', 'hf_vcf_received')
    op.drop_column('genomic_gc_validation_metrics', 'vcf_tbi_received')
    op.drop_column('genomic_gc_validation_metrics', 'raw_vcf_received')
    op.drop_column('genomic_gc_validation_metrics', 'idat_red_md5_received')
    op.drop_column('genomic_gc_validation_metrics', 'hf_vcf_tbi_received')
    op.drop_column('genomic_gc_validation_metrics', 'idat_green_received')
    # ### end Alembic commands ###

def downgrade_rdr():
    # ### commands auto generated by Alembic - please adjust! ###
    op.add_column('genomic_gc_validation_metrics', sa.Column('idat_green_received', mysql.SMALLINT(display_width=6), autoincrement=False, nullable=False))
    op.add_column('genomic_gc_validation_metrics', sa.Column('hf_vcf_tbi_received', mysql.SMALLINT(display_width=6), autoincrement=False, nullable=False))
    op.add_column('genomic_gc_validation_metrics', sa.Column('idat_red_md5_received', mysql.SMALLINT(display_width=6), autoincrement=False, nullable=False))
    op.add_column('genomic_gc_validation_metrics', sa.Column('raw_vcf_received', mysql.SMALLINT(display_width=6), autoincrement=False, nullable=False))
    op.add_column('genomic_gc_validation_metrics', sa.Column('vcf_tbi_received', mysql.SMALLINT(display_width=6), autoincrement=False, nullable=False))
    op.add_column('genomic_gc_validation_metrics', sa.Column('hf_vcf_received', mysql.SMALLINT(display_width=6), autoincrement=False, nullable=False))
    op.add_column('genomic_gc_validation_metrics', sa.Column('cram_md5_received', mysql.SMALLINT(display_width=6), autoincrement=False, nullable=False))
    op.add_column('genomic_gc_validation_metrics', sa.Column('idat_green_md5_received', mysql.SMALLINT(display_width=6), autoincrement=False, nullable=False))
    op.add_column('genomic_gc_validation_metrics', sa.Column('hf_vcf_md5_received', mysql.SMALLINT(display_width=6), autoincrement=False, nullable=False))
    op.add_column('genomic_gc_validation_metrics', sa.Column('gvcf_md5_received', mysql.SMALLINT(display_width=6), autoincrement=False, nullable=False))
    op.add_column('genomic_gc_validation_metrics', sa.Column('gvcf_received', mysql.SMALLINT(display_width=6), autoincrement=False, nullable=False))
    op.add_column('genomic_gc_validation_metrics', sa.Column('cram_received', mysql.SMALLINT(display_width=6), autoincrement=False, nullable=False))
    op.add_column('genomic_gc_validation_metrics', sa.Column('crai_received', mysql.SMALLINT(display_width=6), autoincrement=False, nullable=False))
    op.add_column('genomic_gc_validation_metrics', sa.Column('vcf_received', mysql.SMALLINT(display_width=6), autoincrement=False, nullable=False))
    op.add_column('genomic_gc_validation_metrics', sa.Column('idat_red_received', mysql.SMALLINT(display_width=6), autoincrement=False, nullable=False))
    op.add_column('genomic_gc_validation_metrics', sa.Column('raw_vcf_md5_received', mysql.SMALLINT(display_width=6), autoincrement=False, nullable=False))
    op.add_column('genomic_gc_validation_metrics', sa.Column('raw_vcf_tbi_received', mysql.SMALLINT(display_width=6), autoincrement=False, nullable=False))
    op.add_column('genomic_gc_validation_metrics', sa.Column('vcf_md5_received', mysql.SMALLINT(display_width=6), autoincrement=False, nullable=False))
    # ### end Alembic commands ###

def upgrade_metrics():
    # ### commands auto generated by Alembic - please adjust! ###
    pass
    # ### end Alembic commands ###

def downgrade_metrics():
    # ### commands auto generated by Alembic - please adjust! ###
    pass
    # ### end Alembic commands ###


text:
import ProjectManager from "../../objects/ProjectManager.js";
import User from "../../objects/User.js";
import CameraShot from "../../objects/CameraShot.js";
import { expect } from "chai";
import fs from "fs";

describe("ProjectManager Creation", () => {
    it("Should be able to create a ProjectManager object", done => {
        const projectManager = new ProjectManager();
        expect(projectManager).to.not.be.null;
        done();
    });

    try {
        fs.accessSync(`${__dirname}/../../project-scp-files/project.scp`, fs.F_OK);

        it("Should be able to initialize data from scripting project file", done => {
            ProjectManager.waitForXML(projectManager => {
                expect(projectManager.data).to.not.be.null;
                done();
            });
        });

        it("Can Reload The Project Manager", done => {
            ProjectManager.waitForXML(projectManager => {
                projectManager.data = null;
                projectManager.reloadProject((projectManager) => {
                    expect(projectManager.data).to.not.be.null;
                    done();
                });
            });
        });
    } catch (e) {
        // Then the file doesn't exist, so test for null data
        it("Should not fatally crash on missing project file", done => {
            ProjectManager.waitForXML(projectManager => {
                expect(projectManager.data).to.be.null;
                done();
            });
        });
    } finally {
        it("Can Handle Empty User XML", done => {
            ProjectManager.waitForXML(projectManager => {
                const userArray = projectManager.getUsersFromXML();
                expect(userArray).to.eql([]);
                done();
            });
        });

        it("Can Parse Users From XML", done => {
            const xmlObject = [
                {
                    user: [{
                        chosenTimelines: [{ chosenTimeline: ["0", "1"] }],
                        name: ["John"],
                        roleValue: ["1"],
                    }],
                }
            ];
            ProjectManager.waitForXML(projectManager => {
                const users = projectManager.getUsersFromXML(xmlObject);
                expect(users).to.not.be.empty;
                done();
            });
        });

        it("Can Parse A List of Users To XML", done => {
            const user = new User(0, "John", [], 1);
            ProjectManager.waitForXML(projectManager => {
                const userXML = projectManager.usersToXML([user]);
                expect(userXML[0].user).to.exist;
                expect(userXML[0].user[0]).to.deep.equal(user.toXML());
                done();
            });
        });

        it("Can Return Min and Max Counts Of 0 Shots", done => {
            ProjectManager.waitForXML(projectManager => {
                const minMaxCount = projectManager.getMaxAndMinCount([]);
                expect(minMaxCount.minCount).to.equal(0);
                expect(minMaxCount.maxCount).to.equal(0);
                done();
            });
        });

        it("Can Return Min and Max Counts With Multiple Shots", done => {
            ProjectManager.waitForXML(projectManager => {
                const minMaxCount = projectManager.getMaxAndMinCount([
                    new CameraShot(0, 1, "Shot 1", "Zoom in on director"),
                    new CameraShot(12, 13, "Shot 2", "Gallery Pan"),
                ]);
                expect(minMaxCount.minCount).to.equal(0);
                expect(minMaxCount.maxCount).to.equal(13);
                done();
            });
        });

        it("Can Handle Missing Camera Timelines", done => {
            ProjectManager.waitForXML(projectManager => {
                const cameraTimelines = projectManager.getCameraTimelinesFromXML([{}]);
                expect(cameraTimelines).to.be.empty;
                done();
            });
        });

        it("Can Fetch Preset Updates", done => {
            ProjectManager.waitForPresetUpdates(projectManager => {
                expect(projectManager.initialized).to.be.true;
                done();
            });
        });
    }
});

describe("ProjectManager Async", () => {
    try {
        fs.accessSync(`${__dirname}/../../project-scp-files/project.scp`, fs.F_OK);

        before(done => {
            // Ensure Project Data Loaded Before Running Tests
            ProjectManager.waitForXML(projectManager => done());
        });

        it("Should return a Singleton Object", done => {
            const projectManager1 = new ProjectManager();
            const projectManager2 = new ProjectManager();
            expect(projectManager1).to.equal(projectManager2);
            done();
        });

        it("Should be able to filter timelines", done => {
            const projectManager = new ProjectManager();
            const filteredTimelines = projectManager.filterTimelines([0], projectManager.data);
            expect(filteredTimelines.scriptingProject.cameraTimelines).to.not.be.empty;
            done();
        });
    } catch (e) {
        // Can't run tests on empty data, so unfortunately do nothing here
    }
});


text:
GAIAInfoUpdateServiceFactory::GAIAInfoUpdateServiceFactory()
    : ProfileKeyedServiceFactory("GAIAInfoUpdateService",
                                 ProfileDependencyManager::GetInstance()) {
}

GAIAInfoUpdateServiceFactory::~GAIAInfoUpdateServiceFactory() {}

// static
GAIAInfoUpdateService* GAIAInfoUpdateServiceFactory::GetForProfile(
    Profile* profile) {
  return static_cast<GAIAInfoUpdateService*>(
      GetInstance()->GetServiceForProfile(profile, true));
}

// static
GAIAInfoUpdateServiceFactory* GAIAInfoUpdateServiceFactory::GetInstance() {
  return Singleton<GAIAInfoUpdateServiceFactory>::get();
}

ProfileKeyedService* GAIAInfoUpdateServiceFactory::BuildServiceInstanceFor(
    content::BrowserContext* context) const {
  Profile* profile = static_cast<Profile*>(context);
  if (!GAIAInfoUpdateService::ShouldUseGAIAProfileInfo(profile))
    return NULL;
  return new GAIAInfoUpdateService(profile);
}

void GAIAInfoUpdateServiceFactory::RegisterUserPrefs(
    user_prefs::PrefRegistrySyncable* prefs) {
  prefs->RegisterInt64Pref(prefs::kProfileGAIAInfoUpdateTime,
                           0,
                           user_prefs::PrefRegistrySyncable::UNSYNCABLE_PREF);
  prefs->RegisterStringPref(prefs::kProfileGAIAInfoPictureURL,
                            std::string(),
                            user_prefs::PrefRegistrySyncable::UNSYNCABLE_PREF);
}

bool GAIAInfoUpdateServiceFactory::ServiceIsNULLWhileTesting() const {
  return true;
}