text:
'use strict';

var argv = require('yargs').argv;
var browserify = require('browserify');
var browserifyCss = require('browserify-css');
var markedify = require('markedify');
var buffer = require('vinyl-buffer');

// =======================================================================
// Gulp Plugins
// =======================================================================
var gulp = require('gulp'),
    connect = require('gulp-connect'),
    gutil = require('gulp-util'),
    livereload = require('gulp-livereload'),
    merge = require('merge'),
    rename = require('gulp-rename'),
    jshint = require('gulp-jshint'),
    stylish = require('jshint-stylish'),
    jscs = require('gulp-jscs'),
    concat = require('gulp-concat'),
    streamify = require('gulp-streamify'),
    uglify = require('gulp-uglify'),
    less = require('gulp-less'),
    cssnano = require('gulp-cssnano'),
    prefix = require('gulp-autoprefixer'),
    notify = require('gulp-notify'),
    webserver = require('gulp-webserver'),
    watchify = require('watchify'),
    del = require('del'),
    source = require('vinyl-source-stream'),
    sourceMaps = require('gulp-sourcemaps'),
    runSequence = require('run-sequence'),
    Karma = require('karma').Server,
    gulpif = require('gulp-if'),
    envify = require('envify'),
    ngAnnotate = require('gulp-ng-annotate');

// =======================================================================
// File Paths
// =======================================================================
var filePath = require('./gulp.config')();

// =======================================================================
// Error Handling
// =======================================================================
function handleError(err) {
    gutil.log(err.message);
    // this.emit('end');
}

// =======================================================================
// Server Task
// =======================================================================
var express = require('express'),
    server = express();
// Server settings
server.use(express.static(filePath.build.dest));
// Redirects everything back to our index.html
server.all('/*', function (req, res) {
    res.sendfile('/', {
        root: filePath.build.dest
    });
});
// uncomment the "middleware" section when you are ready to connect to an API
gulp.task('server', function () {
    connect.server({
        root: filePath.build.dest,
        fallback: filePath.build.dest + '/index.html',
        port: 5000,
        livereload: true
        // ,
        // middleware: function(connect, o) {
        //     return [ (function() {
        //         var url = require('url');
        //         var proxy = require('proxy-middleware');
        //         var options = url.parse('http://localhost:3000/'); // path to your dev API
        //         options.route = '/api';
        //         return proxy(options);
        //     })() ];
        // }
    });
});

// =======================================================================
// Clean out dist folder contents on build
// =======================================================================
gulp.task('clean-dev', function () {
    del(['./dist/*.js',
        './dist/css/*.css',
        '!./dist/fonts/**',
        '!./dist/vendor.js',
        '!./dist/vendor.css',
        './dist/*.html',
        './dist/*.png',
        './dist/*.ico',
        './reports/**/*',
        './reports'
    ]);
});

gulp.task('clean-full', function () {
    del(['./dist/*',
        './reports/**/*',
        './reports'
    ]);
});

// =======================================================================
// JSHint
// =======================================================================
gulp.task('lint', function () {
    return gulp.src(filePath.lint.src)
        .pipe(jshint())
        .pipe(jshint.reporter(stylish));
});

// =======================================================================
// Javascript Checkstyles (JSCS)
// =======================================================================
gulp.task('checkstyle', function () {
    return gulp.src(filePath.lint.src)
        .pipe(jscs())
        .on('error', handleError);
});

// =======================================================================
// Browserify Bundle
// =======================================================================

gulp.task('bundle', function () {
    return browserify({
            entries: filePath.browserify.src,
            debug: true
        })
        .transform(browserifyCss, {
            global: true,
        })
        .transform(markedify)
        .bundle() // Start bundle
        .pipe(source(filePath.browserify.src)) // Entry point
        .pipe(buffer()) // Convert to gulp pipeline
        .pipe(sourceMaps.init({
            loadMaps: true
        })) // Strip inline source maps
            .pipe(ngAnnotate())
            .pipe(uglify({
                mangle: false
            })).on('error', handleError)
            .pipe(rename(filePath.browserify.outputFile)) // Rename output from 'app.js' to 'bundle.js'
        .pipe(sourceMaps.write(filePath.browserify.mapDir)) // Save source maps to their own directory
        .pipe(gulp.dest(filePath.browserify.outputDir)) // Save 'bundle' to build/
        .pipe(connect.reload()) // Reload browser if relevant
});

// =======================================================================
// Styles Task
// =======================================================================
gulp.task('styles-dev', function () {
    return gulp.src(filePath.styles.src)
        .pipe(sourceMaps.init())
        .pipe(less())
        .on('error', handleError)
        .pipe(sourceMaps.write())
        .pipe(gulp.dest(filePath.build.dest + '/css'))
        .on('error', handleError)
        .pipe(notify({
            message: 'Styles task complete'
        }))
        .pipe(connect.reload());
});

gulp.task('styles-prod', function () {
    return gulp.src(filePath.styles.src)
        .pipe(less())
        .on('error', handleError)
        .pipe(prefix('last 1 version', '> 1%', {
            map: true
        }))
        .pipe(cssnano())
        .pipe(gulp.dest(filePath.build.dest + '/css'))
        .on('error', handleError)
        .pipe(notify({
            message: 'Styles task complete'
        }));
});

// =======================================================================
// Images Task
// =======================================================================
gulp.task('images', function () {
    return gulp.src(filePath.assets.images.src)
        .on('error', handleError)
        .pipe(gulp.dest(filePath.assets.images.dest))
        .pipe(notify({
            message: 'Images copied'
        }))
        .pipe(connect.reload());
});

// =======================================================================
// Fonts Task
// =======================================================================
gulp.task('fonts', function () {
    return gulp.src(filePath.assets.fonts.src)
        .on('error', handleError)
        .pipe(gulp.dest(filePath.assets.fonts.dest))
        .pipe(connect.reload());
});

// =======================================================================
// Vendor CSS Task
// =======================================================================
//gulp.task('vendorCSS', function() {
//    return gulp.src(filePath.vendorCSS.src)
//    .pipe(concat('vendor.css'))
//    .on('error', handleError)
//    .pipe(cssnano())
//    .pipe(gulp.dest(filePath.build.dest))
//    .pipe(notify({
//        message: 'VendorCSS task complete'
//    }))
//    .pipe(connect.reload());
//});

// =======================================================================
// Copy index.html
// =======================================================================
gulp.task('copyIndex', function () {
    return gulp.src(filePath.copyIndex.src)
        .pipe(gulp.dest(filePath.build.dest))
        .pipe(notify({
            message: 'index.html successfully copied'
        }))
        .pipe(connect.reload());
});

// =======================================================================
// Copy Favicon
// =======================================================================
gulp.task('copyFavicon', function () {
    return gulp.src(filePath.copyFavicon.src)
        .pipe(gulp.dest(filePath.build.dest))
        .pipe(notify({
            message: 'favicon successfully copied'
        }));
});

// =======================================================================
// Watch for changes
// =======================================================================
gulp.task('watch', function () {
    gulp.watch(filePath.styles.watch, ['styles-dev']);
    gulp.watch(filePath.assets.images.watch, ['images']);
    gulp.watch(filePath.browserify.watch, ['bundle']);
    // gulp.watch(filePath.vendorJS.src, ['vendorJS']);
    gulp.watch(filePath.copyIndex.watch, ['copyIndex']);
    gulp.watch(filePath.lint.src, ['checkstyle']);
    console.log('Watching...');
});

// =======================================================================
// Start a webserver to serve the assets to developers directly
// =======================================================================
gulp.task('webserver', function () {
    gulp.src('app')
        .pipe(webserver({
            livereload: true,
            directoryListing: true,
            open: true
        }));
});

// =======================================================================
// Karma Configuration
// =======================================================================
gulp.task('karma', function (done) {
    var karma = new Karma({
        configFile: __dirname + '/karma.conf.js',
        singleRun: !argv.watch
    }, done);
    karma.start();
});

// =======================================================================
// Sequential Build Rendering
// =======================================================================

// run "gulp" in terminal to build the DEV app
gulp.task('build-dev', function (callback) {
    runSequence(
        ['clean-dev', 'lint', 'checkstyle'],
        // images and vendor tasks are removed to speed up build time. Use "gulp build" to do a full re-build of the dev app.
        [
            'bundle',
            'styles-dev', 'copyIndex', 'copyFavicon'
        ], ['server', 'watch'],
        callback
    );
});

// run "gulp test" in terminal to build the DEV app
gulp.task('build-test', function (callback) {
    runSequence(
        ['clean-full', 'lint', 'checkstyle'], ['karma'],
        callback
    );
});

// run "gulp prod" in terminal to build the PROD-ready app
gulp.task('build-prod', function (callback) {
    runSequence(
        ['clean-full', 'lint', 'checkstyle'], [
            'bundle',
            'styles-prod', 'images', 'fonts', 'copyIndex', 'copyFavicon'
        ],
        callback
    );
});

// run "gulp build" in terminal for a full re-build in DEV
gulp.task('build', function (callback) {
    runSequence(
        ['clean-full', 'lint', 'checkstyle'], ['bundle', 'styles-dev', 'images', 'fonts', 'copyIndex', 'copyFavicon'], ['server', 'watch'],
        callback
    );
});

gulp.task('default', ['build-dev']);
gulp.task('test', ['build-test']);
gulp.task('prod', ['build-prod']);


text:
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
  <component name="ProjectModuleManager">
    <modules>
      <module fileurl="file://$PROJECT_DIR$/SchoolStatsinIndia.iml" filepath="$PROJECT_DIR$/SchoolStatsinIndia.iml" />
      <module fileurl="file://$PROJECT_DIR$/app/app.iml" filepath="$PROJECT_DIR$/app/app.iml" />
    </modules>
  </component>
</project>

redpajama_set_name: RedPajamaGithub


text: import filters


text:
Package.describe({
  name: 'ostrio:files',
  version: '1.14.2',
  summary: 'Upload files to Meteor application, with 3rd party storage support: AWS:S3, GridFS and other',
  git: 'https://github.com/VeliovGroup/Meteor-Files',
  documentation: 'README.md'
});

Npm.depends({
  'fs-extra': '9.0.0',
  'request-libcurl': '2.2.0',
  'file-type': '14.2.0',
  eventemitter3: '4.0.0'
});

Package.onUse(function(api) {
  api.versionsFrom('1.9');
  api.use('webapp', 'server');
  api.use(['reactive-var', 'tracker', 'http', 'ddp-client'], 'client');
  api.use(['mongo', 'check', 'random', 'ecmascript', 'ostrio:cookies@2.6.0'], ['client', 'server']);
  api.addAssets('worker.min.js', 'client');
  api.mainModule('server.js', 'server');
  api.mainModule('client.js', 'client');
  api.export('FilesCollection');
});

Package.onTest(function(api) {
  api.use('tinytest');
  api.use(['ecmascript', 'ostrio:files'], ['client', 'server']);
  api.addFiles('tests/helpers.js', ['client', 'server']);
});


text:
require 'rails_helper'

module RailsLti2Provider
  RSpec.describe Registration, :type => :model do
    pending "add some examples to (or delete) #{__FILE__}"
  end
end


text:
.. only:: not (epub or latex or html)

    WARNING: You are looking at unreleased Cilium documentation.
    Please use the official rendered version released here:
    https://docs.cilium.io

.. _envoy:

=====
Envoy
=====

*************
Go Extensions
*************

.. note:: This feature is currently in beta phase.

This is a guide for developers who are interested in writing a Go extension to the
Envoy proxy as part of Cilium.

.. image:: _static/proxylib_logical_flow.png

As depicted above, this framework allows a developer to write a small amount of Go
code (green box) focused on parsing a new API protocol, and this Go code is able to
take full advantage of Cilium features including high-performance redirection to/from Envoy,
rich L7-aware policy language
and access logging, and visibility into encrypted traffic via kTLS (coming soon!).
In sum, you as the developer need only worry about the logic of parsing the protocol,
and Cilium + Envoy + eBPF do the heavy-lifting.

This guide uses simple examples based on a hypothetical "r2d2" protocol
(see `proxylib/r2d2/r2d2parser.go <https://github.com/cilium/cilium/blob/master/proxylib/r2d2/r2d2parser.go>`_)
that might be used to talk to a simple protocol droid a long time ago in a galaxy far, far away.
But it also points to other real protocols like Memcached and Cassandra that already exist in the cilium/proxylib
directory.

Step 1: Decide on a Basic Policy Model
======================================

To get started, take some time to think about what it means to provide protocol-aware security
in the context of your chosen protocol.   Most protocols follow a common pattern of a client
who performs an ''operation'' on a ''resource''.   For example:

- A standard RESTful HTTP request has a GET/POST/PUT/DELETE methods (operation) and URLs (resource).
- A database protocol like MySQL has SELECT/INSERT/UPDATE/DELETE actions (operation) on a combined database + table name (resource).
- A queueing protocol like Kafka has produce/consume (operation) on a particular queue (resource).

A common policy model is to allow the user to whitelist certain operations on one or more resources.
In some cases, the resources need to support regexes to avoid explicit matching on variable content
like ids (e.g., /users/<uuid> would match /users/.*)

In our examples, the ''r2d2'' example, we'll use a basic set of operations (READ/WRITE/HALT/RESET).
The READ and WRITE commands also support a 'filename' resource, while HALT and RESET have no resource.

Step 2: Understand Protocol, Encoding, Framing and Types
========================================================

Next, get your head wrapped around how a protocol looks terms of the raw data, as this is what you'll be parsing.

Try looking for official definitions of the protocol or API.   Official docs will not only help you quickly
learn how the protocol works, but will also help you by documenting tricky corner cases that wouldn't be
obvious just from regular use of the protocol.   For example, here are example specs for
`Redis Protocol <https://redis.io/topics/protocol>`_ , `Cassandra Protocol <https://github.com/apache/cassandra/blob/trunk/doc/native_protocol_v4.spec>`_,
and `AWS SQS <https://docs.aws.amazon.com/AWSSimpleQueueService/latest/APIReference/Welcome.html>`_ .

These specs help you understand protocol aspects like:

- **encoding / framing** : how to recognize the beginning/end of individual requests/replies within a TCP stream.
  This typically involves reading a header that encodes the overall request length, though some simple
  protocols use a delimiter like ''\r\n\'' to separate messages.

- **request/reply fields** : for most protocols, you will need to parse out fields at various offsets
  into the request data in order to extract security-relevant values for visibility + filtering.  In some cases, access
  control requires filtering requests from clients to servers, but in some cases, parsing replies will also be required
  if reply data is required to understand future requests (e.g., prepared-statements in database protocols).

- **message flow** : specs often describe various dependencies between different requests.  Basic protocols tend to
  follow a simple serial request/reply model, but more advanced protocols will support pipelining (i.e., sending
  multiple requests before any replies have been received).

- **protocol errors** : when a Cilium proxy denies a request based on policy, it should return a protocol-specific
  error to the client (e.g., in HTTP, a proxy should return a ''403 Access Denied'' error).  Looking at the protocol
  spec will typically indicate how you should return an equivalent ''Access Denied'' error.

Sometimes, the protocol spec does not give you a full sense of the set of commands that can be sent over the protocol.  In that
case, looking at higher-level user documentation can fill in some of these knowledge gaps.  Here are examples for
`Redis Commands <https://redis.io/commands>`_ and `Cassandra CQL Commands <https://docs.datastax.com/en/archived/cql/3.1/cql/cql_reference/cqlCommandsTOC.html>`_ .

Another great trick is to use `Wireshark <https://www.wireshark.org>`_  to capture raw packet data between
a client and server.   For many protocols, the `Wireshark Sample Captures <https://wiki.wireshark.org/SampleCaptures>`_
has already saved captures for us.  Otherwise, you can easily use tcpdump to capture a file.  For example, for
MySQL traffic on port 3306, you could run the following in a container running the MySQL client or server:
"tcpdump -s 0 port 3306 -w mysql.pcap".  `More Info <https://linuxexplore.com/2012/06/07/use-tcpdump-to-capture-in-a-pcap-file-wireshark-dump/>`_

In our example r2d2 protocol, we'll keep the spec as simple as possible.  It is a text-only based protocol,
with each request being a line terminated by ''\r\n''.  A request starts with a case-insensitive string
command ("READ","WRITE","HALT","RESET").   If the command is "READ" or "WRITE", the command must be followed
by a space, and a non-empty filename that contains only non whitespace ASCII characters.

Step 3: Search for Existing Parser Code / Libraries
===================================================

Look for open source Go library/code that can help.
Is there existing open source Go code that parse your protocol that you can leverage,
either directly as library or a motivating example?  For example, the `tidwall/recon library
<https://github.com/tidwall/redcon>`_ parses Redis in Go, and `Vitess
<https://github.com/vitessio/vitess>`_ parses MySQL in Go.   `Wireshark dissectors
<https://github.com/boundary/wireshark/tree/master/epan/dissectors>`_ also has a wealth of
protocol parsers written in C that can serve as useful guidance.    Note:  finding client-only
protocol parsing code is typically less helpful than finding a proxy implementation, or a full
parser library.   This is because the set of requests a client parsers is typically the inverse
set of the requests a Cilium proxy needs to parse, since the proxy mimics the server rather than
the client.   Still, viewing a Go client can give you a general idea of how to parse the
general serialization format of the protocol.

Step 4: Follow the Cilium Developer Guide
=========================================

It is easiest to start Cilium development by following the :ref:`dev_guide`

After cloning Cilium:

::

    $ cd cilium
    $ contrib/vagrant/start.sh
    $ cd proxylib

While this dev VM is running, you can open additional terminals to the Cilium dev VM
by running ''vagrant ssh'' from within the cilium source directory.

Step 5: Create New Proxy Skeleton
=================================

From inside the proxylib directory, copy the rd2d directory and rename the files.
Replace ''newproto'' with your protocol:

::

    $ mkdir newproto
    $ cd newproto
    $ cp ../r2d2/r2d2parser.go newproto.go
    $ cp ../r2d2/r2d2parser_test.go newproto_test.go

Within both newproto.go and newproto_test.go update references to r2d2 with
your protocol name.   Search for both ''r2d2'' and ''R2D2''.

Also, edit proxylib.go and add the following import line:

::

       _ "github.com/cilium/cilium/proxylib/newproto"

Step 6: Update OnData Method
============================

Implementing a parser requires you as the developer to implement three primary functions,
shown as blue in the diagram below.   We will cover OnData() in this section, and
the other functions in section `Step 9:  Add Policy Loading and Matching`_.

.. image:: _static/proxylib_key_functions.png

The beating heart of your parsing is implementing the onData function.  You can think of any
proxy as have two data streams, one in the request direction (i.e., client to server) and one in
the reply direction (i.e., server to client).   OnData is called when there is data to process,
and the value of the boolean 'reply' parameter indicates the direction of the stream for a given
call to OnData.   The data passed to OnData is a slice of byte slices (i.e., an array of byte arrays).

The return values of the OnData function tell the Go framework tell how data in the stream
should be processed, with four primary outcomes:

- **PASS x** :  The next x bytes in the data stream passed to OnData represent a request/reply that should be
  passed on to the server/client.   The common case here is that this is a request that should be
  allowed by policy, or that no policy is applied.  Note: x bytes may be less than the total amount
  of data passed to OnData, in which case the remaining bytes will still be in the data stream when
  onData is invoked next.  x bytes may also be more than the data that has been passed to OnData.
  For example, in the case of a protocol where the parser filters only on values in a protocol header,
  it is often possible to make a filtering decision, and then pass (or drop) the size of the full
  request/reply without having the entire request passed to Go.

- **MORE x** :  The buffers passed to OnData to do not represent all of the data required to frame and
  filter the request/reply.  Instead, the parser
  needs to see at least x additional bytes beyond the current data to make a decision.
  In some cases, the full request must be read to understand framing and filtering, but in others a decision
  can be made simply by reading a protocol header.   When parsing data, be defensive, and recognize that it is technically possible that
  data arrives one byte at a time. Two common scenarios exist here:

  - **Text-based Protocols** : For text-based protocols
    that use a delimiter like "\r\n", it is common to simply check if the delimiter exists, and return
    MORE 1 if it does not, as technically one more character could result in the delimiter being present.
    See the sample r2d2 parser as a basic example of this.

  - **Binary-based protocols** : Many binary protocols
    have a fixed header length, which containers a field that then indicates the remaining length
    of the request.  In the binary case, first check to make sure a full header is received.  Typically
    the header will indicate both the full request length (i.e., framing), as well as the request type,
    which indicates how much of the full request must be read in order to perform filtering (in many cases, this is less than
    the full request).  A binary parser will typically return MORE if the data passed to OnData is less than
    the header length.   After reading a full header, the simple approach is for the parser to return MORE to wait
    for the full request to be received and parsed  (see the existing CassandraParser as an example).
    However, as an optimization, the parser can attempt to only
    request the minimum number of bytes required beyond the header to make a policy decision, and then PASS or DROP
    the remaining bytes without requiring them to be passed to the Go parser.

- **DROP x** :  Remove the first x bytes from the data stream passed to OnData, as they represent a request/reply
  that should not be forwarded to the client or server based on policy.  Don't worry about making onData return
  a drop right away, as we'll return to DROP in a later step below.

- **ERROR y** : The connection contains data that does not match the protocol spec, and prevents you from further
  parsing the data stream.   The framework will terminate the connection.   An example would be a request length
  that falls outside the min/max specified by the protocol spec, or values for a field that fall outside the values
  indicated by the spec (e.g., wrong versions, unknown commands).  If you are still able to properly frame the
  requests, you can also choose to simply drop the request and return a protocol error (e.g., similar to an
  ''HTTP 400 Bad Request'' error.   But in all cases, you should write your parser defensively, such that you
  never forward a request that you do not understand, as such a request could become an avenue for subverting
  the intended security visibility and filtering policies.  See proxylib/types.h for the set of valid error codes.

See proxylib/proxylib/parserfactory.go for the official OnData interface definition.

Keep it simple, and work iteratively.  Start out just getting the framing right.  Can you write a parser that just
prints out the length and contents of a request, and then PASS each request with no policy enforcement?

One simple trick is to comment out the r2d2 parsing logic in OnData, but leave it in the file as a reference, as your protocol will likely
require similar code as we add more functionality below.

Step 7: Use Unit Testing To Drive Development
=============================================

Use unit tests to drive your development.    Its tempting to want to first test your parser by firing up a
client and server and developing on the fly.   But in our experience you'll iterate faster by using the
great unit test framework created along with the Go proxy framework.   This framework lets you pass
in an example set of requests as byte arrays to a CheckOnDataOK method, which are passed to the parser's OnData method.
CheckOnDataOK takes a set of expected return values, and compares them to the actual return values from OnData
processing the byte arrays.

Take some time to look at the unit tests for the r2d2 parser, and then for more complex parsers like Cassandra
and Memcached.   For simple text-based protocols, you can simply write ASCII strings to represent protocol messages,
and convert them to []byte arrays and pass them to CheckOnDataOK.   For binary protocols, one can either create
byte arrays directly, or use a mechanism to convert a hex string to byte[] array using a helper function like
hexData in cassandra/cassandraparser_test.go

A great way to get the exact data to pass in is to copy the data from the Wireshark captures mentioned
above in Step #2.   You can see the full application layer data streams in Wireshark by right-clicking
on a packet and selecting "Follow As… TCP Stream".  If the protocol is text-based, you can copy the data
as ASCII (see r2d2/r2d2parser_test.go as an example of this).   For binary data, it can be easier to instead
select "raw" in the drop-down, and use a basic utility to convert from ascii strings to binary raw data (see
cassandra/cassandraparser_test.go for an example of this).

To run the unit tests, go to proxylib/newproto and run:

::

  $ go test

This will build the latest version of your parser and unit test files and run the unit tests.

Step 8: Add More Advanced Parsing
=================================

Thinking back to step #1, what are the critical fields to parse out of the request in order to
understand the "operation" and "resource" of each request.  Can you print those out for each request?

Use the unit test framework to pass in increasingly complex requests, and confirm that the parser prints out the right values, and that the
unit tests are properly slicing the datastream into requests and parsing out the required fields.

A couple scenarios to make sure your parser handles properly via unit tests:

- data chunks that are less than a full request (return MORE)
- requests that are spread across multiple data chunks. (return MORE ,then PASS)
- multiple requests that are bundled into a single data chunk (return PASS, then another PASS)
- rejection of malformed requests (return ERROR).

For certain advanced cases, it is required for a parser to store state across requests.
In this case, data can be stored using data structures that
are included as part of the main parser struct.  See CassandraParser in cassandra/cassandraparser.go as an example
of how the parser uses a string to store the current 'keyspace' in use, and uses Go maps to keep
state required for handling prepared queries.

Step 9:  Add Policy Loading and Matching
========================================

Once you have the parsing of most protocol messages ironed out, its time to start enforcing policy.

First, create a Go object that will represent a single rule in the policy language. For example,
this is the rule for the r2d2 protocol, which performs exact match on the command string, and a regex
on the filename:

::

 type R2d2Rule struct {
    cmdExact   string
    fileRegexCompiled *regexp.Regexp
 }

There are two key methods to update:

- Matches :   This function implements the basic logic of comparing data from a single request
  against a single policy rule, and return true if that rule matches (i.e., allows) that request.

- <NewProto>RuleParser : Reads key value pairs from policy, validates those entries, and stores
  them as a <NewProto>Rule object.

See r2d2/r2d2parser.go for examples of both functions for the r2d2 protocol.

You'll also need to update OnData to call p.connection.Matches(), and if this function return false,
return DROP for a request.  Note: despite the similar names between the Matches() function you
create in your newprotoparser.go and p.connection.Matches(), do not confuse
the two.  Your OnData function should always call p.connection.Matches() rather than invoking your
own Matches() directly, as p.connection.Matches()
calls the parser's Matches() function only on the subset of L7 rules that apply for the given
Cilium source identity for this particular connection.

Once you add the logic to call Matches() and return DROP in OnData, you will need to update
unit tests to have policies that allow the traffic you expect to be passed.   The following
is an example of how r2d2/r2d2parser_test.go adds an allow-all policy for a given test:

::

    s.ins.CheckInsertPolicyText(c, "1", []string{`
        name: "cp1"
        policy: 2
        ingress_per_port_policies: <
          port: 80
          rules: <
            l7_proto: "r2d2"
          >
        >
        `})

The following is an example of a policy that would allow READ commands with a file
regex of ".*":

::

    s.ins.CheckInsertPolicyText(c, "1", []string{`
        name: "cp2"
        policy: 2
        ingress_per_port_policies: <
          port: 80
          rules: <
            l7_proto: "r2d2"
            l7_rules: <
            rule: <
              key: "cmd"
              value: "READ"
            >
            rule: <
              key: "file"
              value: ".*"
            >
              >
            >
          >
        >
        `})

Step 10: Inject Error Response
==============================

Simply dropping the request from the request data stream prevents the request from reaching the server, but it would
leave the client hanging, waiting for a response that would never come since the server did not see the request.

Instead, the proxy should return an application-layer reply indicating that access was denied, similar to how
an HTTP proxy would return a ''403 Access Denied'' error.  Look back at the protocol spec discussed in Step 2 to
understand what an access denied message looks like for this protocol, and use the p.connection.Inject() method
to send this error reply back to the client.   See r2d2/r2d2parser.go for an example.

::

    p.connection.Inject(true, []byte("ERROR\r\n"))

Note:  p.connection.Inject() will inject the data it is passed into the reply datastream.  In order for the client
to parse this data correctly, it must be injected at a proper framing boundary (i.e., in between other reply messages
that may be in the reply data stream).  If the client is following a basic serial request/reply model per connection, this is
essentially guaranteed as at the time of a request that is denied, there are no other replies potentially in the
reply datastream.   But if the protocol supports pipelining (i.e., multiple requests in flight) replies must be properly
framed and PASSed on a per request basis, and the timing of the call to p.connection.Inject() must be controlled
such that the client will properly match the Error response with the correct request.   See the Memcached parser
as an example of how to accomplish this.

Step 11: Add Access Logging
===========================

Cilium also has the notion of an ''Access Log'', which records each request handled by the proxy
and indicates whether the request was allowed or denied.

A call to ''p.connection.Log()'' implements access logging. See the OnData function in r2d2/r2d2parser.go
as an example:

::

      p.connection.Log(access_log_entry_type,
        &cilium.LogEntry_GenericL7{
            &cilium.L7LogEntry{
                Proto: "r2d2",
                Fields: map[string]string{
                    "cmd":  reqData.cmd,
                    "file": reqData.file,
                },
            },
      })

Step 12: Manual Testing
=======================

Find the standard docker container for running the protocol server.  Often the same image also has a CLI client that you can use as a client.

Start both a server and client container running in the cilium dev VM, and attach them to the already created "cilium-net".  For example, with Cassandra, we run:

::

    docker run --name cass-server -l id=cass-server -d --net cilium-net cassandra

    docker run --name cass-client -l id=cass-client -d --net cilium-net cassandra sh -c 'sleep 3000'

Note that we run both containers with labels that will make it easy to refer to these containers in a cilium
network policy.   Note that we have the client container run the sleep command, as we will use 'docker exec' to
access the client CLI.

Use ''cilium endpoint list'' to identify the IP address of the protocol server.

::

  $ cilium endpoint list
  ENDPOINT   POLICY (ingress)   POLICY (egress)   IDENTITY   LABELS (source:key[=value])   IPv6                 IPv4            STATUS
             ENFORCEMENT        ENFORCEMENT
  2987       Disabled           Disabled          31423      container:id=cass-server      f00d::a0b:0:0:bab    10.11.51.247    ready
  27333      Disabled           Disabled          4          reserved:health               f00d::a0b:0:0:6ac5   10.11.92.46     ready
  50923      Disabled           Disabled          18253      container:id=cass-client      f00d::a0b:0:0:c6eb   10.11.175.191   ready

One can then invoke the client CLI using that server IP address (10.11.51.247 in the above example):

::

 docker exec -it cass-client sh -c 'cqlsh 10.11.51.247 -e "select * from system.local"'

Note that in the above example, ingress policy is not enforced for the Cassandra server endpoint, so no data will flow through the
Cassandra parser.  A simple ''allow all'' L7 Cassandra policy can be used to send all data to the Cassandra server through the
Go Cassandra parser.  This policy has a single empty rule, which matches all requests.  An allow all policy looks like:

::

  [ {
    "endpointSelector": {"matchLabels":{"id":"cass-server"}},
    "ingress": [ {
	  "toPorts": [{
		  "ports": [{"port": "9042", "protocol": "TCP"}],
            		"rules": {
                		"l7proto": "cassandra",
                		"l7": [{}]
            		}
		}]
	  } ]
  }]

A policy can be imported into cilium using ''cilium policy import'', after which another call to ''cilium endpoint list''
confirms that ingress policy is now in place on the server.  If the above policy was saved to a file cass-allow-all.json,
one would run:

::

    $ cilium policy import cass-allow-all.json
    Revision: 1
    $ cilium endpoint list
    ENDPOINT   POLICY (ingress)   POLICY (egress)   IDENTITY   LABELS (source:key[=value])   IPv6                 IPv4            STATUS
               ENFORCEMENT        ENFORCEMENT
    2987       Enabled            Disabled          31423      container:id=cass-server      f00d::a0b:0:0:bab    10.11.51.247    ready
    27333      Disabled           Disabled          4          reserved:health               f00d::a0b:0:0:6ac5   10.11.92.46     ready
    50923      Disabled           Disabled          18253      container:id=cass-client      f00d::a0b:0:0:c6eb   10.11.175.191   ready

Note that policy is now showing as ''Enabled'' for the Cassandra server on ingress.

To remove this or any other policy, run:

::

    $ cilium policy delete --all

To install a new policy, first delete, and then run ''cilium policy import'' again.  For example, the following policy would allow
select statements on a specific set of tables to this Cassandra server, but deny all other queries.

::

  [ {
    "endpointSelector": {"matchLabels":{"id":"cass-server"}},
    "ingress": [ {
          "toPorts": [{
                  "ports": [{"port": "9042", "protocol": "TCP"}],
                        "rules": {
                                "l7proto": "cassandra",
                                "l7": [
                                       { "query_action" : "select", "query_table": "^system.*"},
                                       { "query_action" : "select", "query_table" : "^posts_db.posts$"}

                                ]}
                        }]
         }]
  } ]

When performing manual testing, remember that each time you change your Go proxy code, you must
re-run ''make'' and ''sudo make install'' and then restart the cilium-agent process.  If the only changes
you have made since last compiling cilium are in your cilium/proxylib directory, you can safely
just run ''make'' and ''sudo make install''  in that directory, which saves time.
For example:

::

  $ cd proxylib  // only safe is this is the only directory that has changed
  $ make
    <snip>
  $ sudo make install
    <snip>

If you rebase or other files change, you need to run both commands from the top level directory.

Cilium agent default to running as a service in the development VM.  However, the default options do not include
the ''--debug-verbose=flow'' flag, which is critical to getting visibility in troubleshooting Go proxy frameworks.
So it is easiest to stop the cilium service and run the cilium-agent directly as a command in a terminal window,
and adding the ''--debug-verbose=flow'' flag.

::

  $ sudo service cilium stop

  $ sudo /usr/bin/cilium-agent --debug --ipv4-range 10.11.0.0/16 --kvstore-opt consul.address=192.168.33.11:8500 --kvstore consul -t vxlan --fixed-identity-mapping=128=kv-store --fixed-identity-mapping=129=kube-dns --debug-verbose=flow

Step 13: Add Runtime Tests
==========================

Before submitting this change to the Cilium community, it is recommended that you add runtime tests that will run as
part of Cilium's continuous integration testing.   Usually these runtime test can be based on the same container
images and test commands you used for manual testing.

The best approach for adding runtime tests is typically to start out by copying-and-pasting an existing L7 protocol runtime
test and then updating it to run the container images and CLI commands specific to the new protocol.
See cilium/test/runtime/cassandra.go as an example that matches the use of Cassandra described above in the manual testing
section.   Note that the json policy files used by the runtime tests are stored in cilium/test/runtime/manifests, and
the Cassandra example policies in those directories are easy to use as a based for similar policies you may create for your
new protocol.

Step 14: Review Spec for Corner Cases
=====================================

Many protocols have advanced features or corner cases that will not manifest themselves as part of basic testing.
Once you have written a first rev of the parser, it is a good idea to go back and review the protocol's spec or list of
commands to see what if any aspects may fall outside the scope of your initial parser.
For example, corner cases like the handling of empty or nil lists may not show up in your testing, but may cause your
parser to fail.   Add more unit tests to cover these corner cases.
It is OK for the first rev of your parser not to handle all types of requests, or to have a simplified policy structure
in terms of which fields can be matched.   However, it is
important to know what aspects of the protocol you are not parsing, and ensure that it does not lead to any security concerns.
For example, failing to parse prepared statements in a database protocol and instead just passing PREPARE and EXECUTE
commands through would lead to gaping security whole that would render your other filtering meaningless in the face of
a sophisticated attacker.

Step 15: Write Docs or Getting Started Guide (optional)
=======================================================

At a minimum, the policy examples included as part of the runtime tests serve
as basic documentation of the policy and its expected behavior.  But we also
encourage adding more user friendly examples and documentation, for example,
Getting Started Guides.  cilium/Documentation/gettingstarted/cassandra.rst is
a good example to follow.   Also be sure to update Documentation/gettingstarted/index.rst
with a link to this new getting started guide.

With that, you are ready to post this change for feedback from the Cilium community.  Congrats!


text:
//
//  CMSetUpViewController.m
//  CaibeiMarket
//
//  Created by ccguo on 2017/7/28.
//  Copyright © 2017年 qianji. All rights reserved.
//

#import "CMSettingViewController.h"

#import "XBConst.h"
#import "XBSettingCell.h"
#import "XBSettingItemModel.h"
#import "XBSettingSectionModel.h"
#import "RootWebViewController.h"

#import "CMUserManager.h"

@interface CMSettingViewController ()<UITableViewDelegate,UITableViewDataSource>

@property (nonatomic,strong) NSArray  *sectionArray; /**< section模型数组*/

@end

@implementation CMSettingViewController

- (void)viewDidLoad
{
    [super viewDidLoad];
    self.title = @"设置";
    self.view.backgroundColor = [UIColor whiteColor];
    self.tableView.separatorStyle = UITableViewCellSeparatorStyleNone;
    self.tableView.delegate = self;
    self.tableView.dataSource = self;
    [self.view addSubview:self.tableView];
    [self setupSections];
    [self.tableView reloadData];
}

- (void)showAlert:(NSString *)title
{
    UIAlertView *alert = [[UIAlertView alloc]initWithTitle:@"点击了" message:title delegate:nil cancelButtonTitle:@"确定" otherButtonTitles: nil];
    [alert show];
}

#pragma - mark setup
- (void)setupSections
{
    __weak typeof(self) weakSelf = self;
    //************************************section1
    XBSettingItemModel *item1 = [[XBSettingItemModel alloc]init];
    item1.funcName = @"修改密码";
    item1.executeCode = ^{
        NSLog(@"修改密码");
        [self showAlert:@"修改密码"];
    };
    item1.detailText = @"做任务赢大奖";
    item1.accessoryType = XBSettingAccessoryTypeDisclosureIndicator;

    XBSettingItemModel *item2 = [[XBSettingItemModel alloc]init];
    item2.funcName = @"推送提醒";
    item2.accessoryType = XBSettingAccessoryTypeSwitch;
    item2.switchValueChanged = ^(BOOL isOn)
    {
        NSLog(@"推送提醒开关状态===%@",isOn?@"open":@"close");
        UIAlertView *alert = [[UIAlertView alloc]initWithTitle:@"推送提醒" message:isOn?@"open":@"close" delegate:nil cancelButtonTitle:@"确定" otherButtonTitles: nil];
        [alert show];
    };

    XBSettingSectionModel *section1 = [[XBSettingSectionModel alloc]init];
    section1.sectionHeaderHeight = 18;
    section1.itemArray = @[item1,item2];

    //************************************section2
    XBSettingItemModel *item4 = [[XBSettingItemModel alloc]init];
    item4.funcName = @"给个好评";
    item4.detailImage = [UIImage imageNamed:@"icon-new"];
    item4.accessoryType = XBSettingAccessoryTypeNone;

    XBSettingItemModel *item5 = [[XBSettingItemModel alloc]init];
    item5.funcName = @"意见反馈";
    item5.detailImage = [UIImage imageNamed:@"next_normal"];
    item5.accessoryType = XBSettingAccessoryTypeNone;

    XBSettingItemModel *item6 = [[XBSettingItemModel alloc]init];
    item6.funcName = @"帮助中心";
    item6.detailImage = [UIImage imageNamed:@"next_normal"];
    item6.accessoryType = XBSettingAccessoryTypeNone;
    item6.executeCode = ^{
        NSString * url = @"https://www.icaibei.cn/admin/about/help.html";
        NSString * title = @"帮助中心";
        RootWebViewController * webview = [[RootWebViewController alloc] initWithParams:@{@"title" : title,@"url" : url}];
        [self.navigationController pushViewController:webview animated:YES];

    };

    XBSettingSectionModel *section2 = [[XBSettingSectionModel alloc]init];
    section2.sectionHeaderHeight = 18;
    section2.itemArray = @[item4,item5,item6];

    //************************************section3
    XBSettingItemModel *item7 = [[XBSettingItemModel alloc]init];
    item7.funcName = @"关于我们";
    item7.accessoryType = XBSettingAccessoryTypeNone;
    item7.detailImage = [UIImage imageNamed:@"next_normal"];
    item7.executeCode = ^{
        RootWebViewController * viewController = [[RootWebViewController alloc] initWithUrl:URL_icaibei_about_url];
        [weakSelf.navigationController pushViewController:viewController animated:YES];
    };

    XBSettingItemModel *item8 = [[XBSettingItemModel alloc]init];
    item8.funcName = @"分享到好友";
    item8.accessoryType = XBSettingAccessoryTypeNone;
    item8.executeCode = ^{
        [CMShareManager shareView:weakSelf.view info:nil];
    };

    NSString * version = [[[NSBundle mainBundle] infoDictionary] objectForKey:@"CFBundleShortVersionString"];
    NSString * versionStr = [NSString stringWithFormat:@"V%@",version];
    XBSettingItemModel *item9 = [[XBSettingItemModel alloc]init];
    item9.funcName = @"版本号";
    item9.accessoryType = XBSettingAccessoryTypeNone;
    item9.detailText = versionStr;

    XBSettingSectionModel *section3 = [[XBSettingSectionModel alloc]init];
    section3.sectionHeaderHeight = 18;
    section3.itemArray = @[item7,item8,item9];

    //************************************section4

    XBSettingItemModel *item10 = [[XBSettingItemModel alloc]init];
    item10.funcName = @"退出";
    item10.accessoryType = XBSettingAccessoryTypeCenter;
    item10.executeCode = ^{
        [weakSelf loginOut];
    };

    XBSettingSectionModel *section4 = [[XBSettingSectionModel alloc]init];
    section4.sectionHeaderHeight = 18;
    section4.itemArray = @[item10];

    self.sectionArray = @[section1,section2,section3,section4];
}

- (void)loginOut
{
    kWeakSelf(self)
    [[CMUserManager sharedInstance] logout:^(BOOL success, NSString *des) {
        dispatch_after(dispatch_time(DISPATCH_TIME_NOW, (int64_t)(0.2 * NSEC_PER_SEC)), dispatch_get_main_queue(), ^{
            [weakself.navigationController popToRootViewControllerAnimated:YES];
        });
        NSLog(@"log out");
    }];
}

#pragma mark - Table view data source

- (NSInteger)numberOfSectionsInTableView:(UITableView *)tableView {
    return self.sectionArray.count;
}

- (NSInteger)tableView:(UITableView *)tableView numberOfRowsInSection:(NSInteger)section {
    XBSettingSectionModel *sectionModel = self.sectionArray[section];
    return sectionModel.itemArray.count;
}

- (UITableViewCell *)tableView:(UITableView *)tableView cellForRowAtIndexPath:(NSIndexPath *)indexPath
{
    static NSString *identifier = @"setting";
    XBSettingSectionModel *sectionModel = self.sectionArray[indexPath.section];
    XBSettingItemModel *itemModel = sectionModel.itemArray[indexPath.row];

    XBSettingCell *cell = [tableView dequeueReusableCellWithIdentifier:identifier];
    if (!cell) {
        cell = [[XBSettingCell alloc]initWithStyle:UITableViewCellStyleDefault reuseIdentifier:identifier];
    }
    cell.item = itemModel;
    return cell;
}

#pragma - mark UITableViewDelegate
- (CGFloat)tableView:(UITableView *)tableView heightForHeaderInSection:(NSInteger)section
{
    XBSettingSectionModel *sectionModel = self.sectionArray[section];
    return sectionModel.sectionHeaderHeight;
}

- (void)tableView:(UITableView *)tableView didSelectRowAtIndexPath:(NSIndexPath *)indexPath
{
    [tableView deselectRowAtIndexPath:indexPath animated:YES];
    XBSettingSectionModel *sectionModel = self.sectionArray[indexPath.section];
    XBSettingItemModel *itemModel = sectionModel.itemArray[indexPath.row];
    if (itemModel.executeCode) {
        itemModel.executeCode();
    }
}
- (void)scrollViewDidScroll:(UIScrollView *)scrollView {
    XBSettingSectionModel *sectionModel = [self.sectionArray firstObject];
    CGFloat sectionHeaderHeight = sectionModel.sectionHeaderHeight;

    if (scrollView.contentOffset.y<=sectionHeaderHeight&&scrollView.contentOffset.y>=0) {
        scrollView.contentInset = UIEdgeInsetsMake(-scrollView.contentOffset.y, 0, 0, 0);
    } else if (scrollView.contentOffset.y>=sectionHeaderHeight) {
        scrollView.contentInset = UIEdgeInsetsMake(-sectionHeaderHeight, 0, 0, 0);
    }
}

@end


text:
namespace rocksdb {

struct IngestedFileInfo {
  // External file path
  std::string external_file_path;
  // Smallest user key in external file
  std::string smallest_user_key;
  // Largest user key in external file
  std::string largest_user_key;
  // Sequence number for keys in external file
  SequenceNumber original_seqno;
  // Offset of the global sequence number field in the file, will
  // be zero if version is 1 (global seqno is not supported)
  size_t global_seqno_offset;
  // External file size
  uint64_t file_size;
  // total number of keys in external file
  uint64_t num_entries;
  // total number of range deletions in external file
  uint64_t num_range_deletions;
  // Id of column family this file shoule be ingested into
  uint32_t cf_id;
  // TableProperties read from external file
  TableProperties table_properties;
  // Version of external file
  int version;

  // FileDescriptor for the file inside the DB
  FileDescriptor fd;
  // file path that we picked for file inside the DB
  std::string internal_file_path;
  // Global sequence number that we picked for the file inside the DB
  SequenceNumber assigned_seqno = 0;
  // Level inside the DB we picked for the external file.
  int picked_level = 0;
  // Whether to copy or link the external sst file. copy_file will be set to
  // false if ingestion_options.move_files is true and underlying FS
  // supports link operation. Need to provide a default value to make the
  // undefined-behavior sanity check of llvm happy. Since
  // ingestion_options.move_files is false by default, thus copy_file is true
  // by default.
  bool copy_file = true;

  InternalKey smallest_internal_key() const {
    return InternalKey(smallest_user_key, assigned_seqno,
                       ValueType::kTypeValue);
  }

  InternalKey largest_internal_key() const {
    return InternalKey(largest_user_key, assigned_seqno, ValueType::kTypeValue);
  }
};

class ExternalSstFileIngestionJob {
 public:
  ExternalSstFileIngestionJob(
      Env* env, VersionSet* versions, ColumnFamilyData* cfd,
      const ImmutableDBOptions& db_options, const EnvOptions& env_options,
      SnapshotList* db_snapshots,
      const IngestExternalFileOptions& ingestion_options)
      : env_(env),
        versions_(versions),
        cfd_(cfd),
        db_options_(db_options),
        env_options_(env_options),
        db_snapshots_(db_snapshots),
        ingestion_options_(ingestion_options),
        job_start_time_(env_->NowMicros()),
        consumed_seqno_(false) {}

  // Prepare the job by copying external files into the DB.
  Status Prepare(const std::vector<std::string>& external_files_paths,
                 uint64_t next_file_number, SuperVersion* sv);

  // Check if we need to flush the memtable before running the ingestion job
  // This will be true if the files we are ingesting are overlapping with any
  // key range in the memtable.
  //
  // @param super_version A referenced SuperVersion that will be held for the
  //    duration of this function.
  //
  // Thread-safe
  Status NeedsFlush(bool* flush_needed, SuperVersion* super_version);

  // Will execute the ingestion job and prepare edit() to be applied.
  // REQUIRES: Mutex held
  Status Run();

  // Update column family stats.
  // REQUIRES: Mutex held
  void UpdateStats();

  // Cleanup after successful/failed job
  void Cleanup(const Status& status);

  VersionEdit* edit() { return &edit_; }

  const autovector<IngestedFileInfo>& files_to_ingest() const {
    return files_to_ingest_;
  }

  // Whether to increment VersionSet's seqno after this job runs
  bool ShouldIncrementLastSequence() const { return consumed_seqno_; }

 private:
  // Open the external file and populate `file_to_ingest` with all the
  // external information we need to ingest this file.
  Status GetIngestedFileInfo(const std::string& external_file,
                             IngestedFileInfo* file_to_ingest,
                             SuperVersion* sv);

  // Assign `file_to_ingest` the appropriate sequence number and  the lowest
  // possible level that it can be ingested to according to compaction_style.
  // REQUIRES: Mutex held
  Status AssignLevelAndSeqnoForIngestedFile(SuperVersion* sv,
                                            bool force_global_seqno,
                                            CompactionStyle compaction_style,
                                            IngestedFileInfo* file_to_ingest,
                                            SequenceNumber* assigned_seqno);

  // File that we want to ingest behind always goes to the lowest level;
  // we just check that it fits in the level, that DB allows ingest_behind,
  // and that we don't have 0 seqnums at the upper levels.
  // REQUIRES: Mutex held
  Status CheckLevelForIngestedBehindFile(IngestedFileInfo* file_to_ingest);

  // Set the file global sequence number to `seqno`
  Status AssignGlobalSeqnoForIngestedFile(IngestedFileInfo* file_to_ingest,
                                          SequenceNumber seqno);

  // Check if `file_to_ingest` can fit in level `level`
  // REQUIRES: Mutex held
  bool IngestedFileFitInLevel(const IngestedFileInfo* file_to_ingest,
                              int level);

  Env* env_;
  VersionSet* versions_;
  ColumnFamilyData* cfd_;
  const ImmutableDBOptions& db_options_;
  const EnvOptions& env_options_;
  SnapshotList* db_snapshots_;
  autovector<IngestedFileInfo> files_to_ingest_;
  const IngestExternalFileOptions& ingestion_options_;
  VersionEdit edit_;
  uint64_t job_start_time_;
  bool consumed_seqno_;
};

}  // namespace rocksdb


text:
use anyhow::bail;
use async_trait::async_trait;
use serenity::builder::{
    CreateApplicationCommand, CreateApplicationCommandOption, CreateEmbed, CreateEmbedAuthor,
    CreateInteractionResponse, CreateInteractionResponseData,
};
use serenity::client::Context;
use serenity::model::application::command::CommandOptionType;
use serenity::model::application::interaction::application_command::{
    ApplicationCommandInteraction, CommandData,
};
use serenity::model::application::interaction::InteractionResponseType;
use serenity::model::channel::MessageFlags;
use serenity::model::guild::PartialMember;
use serenity::model::user::User;
use tracing::error;
use typemap_rev::TypeMap;

use crate::config::Config;
use crate::constants::EMBED_COLOR;
use crate::discord::slash_commands::whois::WhoisCommandOptionFailure::MissingOption;
use crate::discord::slash_commands::SlashCommand;
use crate::discord::util::{
    verify_guild_slash_command, CommandDataExt, GuildSlashCommandInteraction,
};
use crate::service::guild::{CachedGuild, GuildService};

pub struct WhoisCommand;

struct WhoisCommandOptions<'a> {
    target: (&'a User, Option<&'a PartialMember>),
}

enum WhoisCommandOptionFailure {
    MissingOption,
}

fn generate_options(data: &CommandData) -> Result<WhoisCommandOptions, WhoisCommandOptionFailure> {
    let target = if let Some(target) = data.user("user") {
        target
    } else {
        return Err(MissingOption);
    };

    Ok(WhoisCommandOptions { target })
}

fn generate_member_embed(guild: &CachedGuild, member: &PartialMember, user: &User) -> CreateEmbed {
    let boost_status = match member.premium_since {
        Some(time) => format!("Since <t:{}>", time.unix_timestamp()),
        None => "Not Boosting".into(),
    };

    let flags = match user.public_flags {
        Some(flags) => format!("{:?}", flags),
        None => "<none>".into(),
    };

    let known_as = match &member.nick {
        Some(nick) => format!("{} - {}", user.tag(), nick),
        None => user.tag(),
    };

    let title = if guild.owner_id == user.id {
        format!("Owner of {}", guild.name)
    } else {
        format!("Member of {}", guild.name)
    };

    let created_at = format!("<t:{}>", user.created_at().unix_timestamp());
    let joined_at = match &member.joined_at {
        Some(time) => format!("<t:{}>", time.unix_timestamp()),
        None => "<unknown>".into(),
    };

    CreateEmbed::default()
        .author(CreateEmbedAuthor::new(known_as).icon_url(user.face()))
        .title(title)
        .field("ID", &user.id.to_string(), false)
        .field("User Flags", &flags, false)
        .field("Registered On", &created_at, true)
        .field("Joined On", &joined_at, true)
        .field("Boost Status", &boost_status, true)
        .colour(EMBED_COLOR)
}

fn generate_user_embed(user: &User) -> CreateEmbed {
    let flags = match user.public_flags {
        Some(flags) => format!("{:?}", flags),
        None => "<none>".into(),
    };

    let created_at = format!("<t:{}>", user.created_at().unix_timestamp());

    CreateEmbed::default()
        .author(CreateEmbedAuthor::new(user.tag()).icon_url(user.face()))
        .title("Discord User")
        .field("ID", &user.id.to_string(), false)
        .field("User Flags", &flags, false)
        .field("Registered On", &created_at, false)
        .colour(EMBED_COLOR)
}

#[async_trait]
impl SlashCommand for WhoisCommand {
    fn command_name(&self) -> &'static str {
        "whois"
    }

    fn create_command(&self) -> CreateApplicationCommand {
        CreateApplicationCommand::new("whois")
            .description("displays information about given user or server member")
            .dm_permission(false)
            .add_option(
                CreateApplicationCommandOption::new(
                    CommandOptionType::User,
                    "user",
                    "target user to query",
                )
                .required(true),
            )
    }

    async fn handle_command(
        &self,
        context: &Context,
        interaction: &ApplicationCommandInteraction,
        _config: &Config,
        services: &TypeMap,
    ) -> anyhow::Result<()> {
        let GuildSlashCommandInteraction {
            guild_id,
            member: _,
            permissions: _,
        } = verify_guild_slash_command(interaction)?;

        let options = match generate_options(&interaction.data) {
            Ok(options) => options,
            Err(MissingOption) => {
                bail!("interaction has missing data options")
            }
        };

        let (user, member_option) = options.target;

        if let Some(member) = member_option {
            let guild_service = if let Some(service) = services.get::<GuildService>() {
                service
            } else {
                bail!("couldn't get guild service!");
            };

            let guild = guild_service.get_guild(guild_id).await?;

            let embed = generate_member_embed(&guild, member, user);

            let data = CreateInteractionResponseData::default()
                .flags(MessageFlags::EPHEMERAL)
                .add_embed(embed);

            let response = CreateInteractionResponse::default()
                .kind(InteractionResponseType::ChannelMessageWithSource)
                .interaction_response_data(data);

            interaction
                .create_interaction_response(&context.http, response)
                .await
                .map_err(|err| {
                    error!("failed to reply to interaction {}", err);
                    err
                })?;
        } else {
            let embed = generate_user_embed(user);

            let data = CreateInteractionResponseData::default()
                .flags(MessageFlags::EPHEMERAL)
                .add_embed(embed);

            let response = CreateInteractionResponse::default()
                .kind(InteractionResponseType::ChannelMessageWithSource)
                .interaction_response_data(data);

            interaction
                .create_interaction_response(&context.http, response)
                .await
                .map_err(|err| {
                    error!("failed to reply to interaction {}", err);
                    err
                })?;
        }

        Ok(())
    }
}


text:
package org.springframework.samples.imagedb.web;

import java.io.IOException;
import java.io.OutputStream;

import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.samples.imagedb.ImageDatabase;
import org.springframework.stereotype.Controller;
import org.springframework.ui.ModelMap;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RequestParam;
import org.springframework.web.multipart.MultipartFile;

/**
 * MultiActionController for the image list/upload UI.
 *
 * @author Juergen Hoeller
 * @since 07.01.2004
 */
@Controller
public class ImageController {

	private final ImageDatabase imageDatabase;

	@Autowired
	public ImageController(ImageDatabase imageDatabase) {
		this.imageDatabase = imageDatabase;
	}

	@RequestMapping("/imageList")
	public String showImageList(ModelMap model) {
		model.addAttribute("images", this.imageDatabase.getImages());
		return "imageList";
	}

	@RequestMapping("/imageContent")
	public void streamImageContent(@RequestParam("name") String name, OutputStream outputStream) throws IOException {
		this.imageDatabase.streamImage(name, outputStream);
	}

	@RequestMapping("/imageUpload")
	public String processImageUpload(
			@RequestParam("name") String name, @RequestParam("description") String description,
			@RequestParam("image") MultipartFile image) throws IOException {

		this.imageDatabase.storeImage(name, image.getInputStream(), (int) image.getSize(), description);
		return "redirect:imageList";
	}

	@RequestMapping("/clearDatabase")
	public String clearDatabase() {
		this.imageDatabase.clearDatabase();
		return "redirect:imageList";
	}

}


text:
package fractals;

import java.awt.image.BufferedImage;

import complex.Complex;
import fractals.colorSchemes.ColorSchemeManager;
import fractals.colorSchemes.ColorSchemeManager.ColorSchemeManagerOptions;
import fractals.colorSchemes.LinearColorScheme;
import fractals.colorSchemes.WaveColorScheme;
import fractals.settings.MandelBrotSettingsManager;
import util.math.Point;
import util.math.Scale;

public class MandelBrotFractal extends AbstractFractal {

	private int maxIterations;
	private float escapeValue;

	public MandelBrotFractal() {

		// Set identification
		identifier = FractalIdentifier.MANDELBROT_SET;

		// Set default values
		name = "Mandelbrot Set";
		maxIterations = 512;
		escapeValue = 2.0f;
		scale = new Scale(new Point(-0.5, 0));

		// Create color scheme manager
		colorSchemeManager = new ColorSchemeManager();

		WaveColorScheme waveColorScheme = new WaveColorScheme();

		LinearColorScheme linearColorScheme = new LinearColorScheme(maxIterations);

		colorSchemeManager.addColorScheme(waveColorScheme, ColorSchemeManagerOptions.SET_AS_ACTIVE);
		colorSchemeManager.addColorScheme(linearColorScheme);

		settingsManager = new MandelBrotSettingsManager(this);

	}

	public int getMaxIterations() {
		return maxIterations;
	}

	public void setMaxIterations(int maxIterations) {

		this.maxIterations = maxIterations;
		((LinearColorScheme) colorSchemeManager.getColorScheme("LinearColorScheme")).setSteps(maxIterations);

	}

	public float getEscapeValue() {
		return escapeValue;
	}

	public void setEscapeValue(float escapeValue) {
		this.escapeValue = escapeValue;
	}

	@Override
	public BufferedImage generateImage(int width, int height, Point[][] points) {

		// Create empty image
		BufferedImage image = new BufferedImage(width, height, BufferedImage.TYPE_INT_RGB);

		for (int x = 0; x < width; x++) {

			for (int y = 0; y < height; y++) {

				Complex constant = new Complex(points[x][y].x, points[x][y].y);

				// Get the escape number
				int escapeNumber = getEscapeValue(constant);

				// Set color
				int colorValue = colorSchemeManager.getActiveColorScheme().getRGBValue(escapeNumber);
				image.setRGB(x, y, colorValue);

			}

		}

		return image;

	}

	public int getEscapeValue(Complex constant) {

		Complex value = new Complex(0, 0);
		int i;

		// Perform calculation
		for (i = 0; i < maxIterations && value.getModulus() < escapeValue; i++) {

			value.power(2).add(constant);

		}

		return i;

	}
}


text:
<?php namespace App\Models\History;

use Illuminate\Database\Eloquent\Model;
use App\Models\History\Traits\Relationship\HistoryRelationship;

/**
 * Class History
 * package App
 */
class History extends Model
{
    use HistoryRelationship;

    /**
     * The database table used by the model.
     *
     * @var string
     */
    protected $table = 'history';

    /**
     * The attributes that are mass assignable.
     *
     * @var array
     */
    protected $fillable = ['type_id', 'user_id', 'entity_id', 'icon', 'class', 'text', 'assets'];
}


text:
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE markables SYSTEM "markables.dtd">
<markables xmlns="www.eml.org/NameSpaces/intensifier">
<markable id="markable_107" span="word_1866" degree="medium"  mmax_level="intensifier"  emo-expression-ref="emo-expression:markable_106" />
<markable id="markable_45" span="word_638" degree="medium"  mmax_level="intensifier"  emo-expression-ref="emo-expression:markable_43" />
<markable id="markable_56" span="word_745" degree="medium"  mmax_level="intensifier"  emo-expression-ref="emo-expression:markable_55" />
</markables>


text:
package com.aventstack.extentreports.reporter;

import java.io.File;

public class FileUtil {
    private FileUtil() {
    }

    public static String getExtension(File f) {
        String name = f.getName();
        int i = name.lastIndexOf('.');
        if (i > 0)
            return name.substring(i + 1);
        return "";
    }

    public static String getExtension(String filePath) {
        return getExtension(new File(filePath));
    }
}


text:
package org.apache.flink.runtime.state.ttl;

import org.apache.flink.api.common.functions.FoldFunction;
import org.apache.flink.api.common.state.StateTtlConfiguration;

/**
 * This class wraps folding function with TTL logic.
 *
 * @param <T> Type of the values folded into the state
 * @param <ACC> Type of the value in the state
 *
 * @deprecated use {@link TtlAggregateFunction} instead
 */
@Deprecated
class TtlFoldFunction<T, ACC>
	extends AbstractTtlDecorator<FoldFunction<T, ACC>>
	implements FoldFunction<T, TtlValue<ACC>> {
	private final ACC defaultAccumulator;

	TtlFoldFunction(
		FoldFunction<T, ACC> original, StateTtlConfiguration config, TtlTimeProvider timeProvider, ACC defaultAccumulator) {
		super(original, config, timeProvider);
		this.defaultAccumulator = defaultAccumulator;
	}

	@Override
	public TtlValue<ACC> fold(TtlValue<ACC> accumulator, T value) throws Exception {
		ACC userAcc = getUnexpired(accumulator);
		userAcc = userAcc == null ? defaultAccumulator : userAcc;
		return wrapWithTs(original.fold(userAcc, value));
	}
}


text:
"""
This example takes a RequestID of an already existing export request, prints
the corresponding "Request URL" and offers to download the available files.
Note that you can also use RequestIDs from export requests, that were
submitted using the JSOC website.
"""
from __future__ import absolute_import, division, print_function
import os
from six.moves import input
import example_helpers
import drms

# Print the doc string of this example.
print(__doc__)

# Export request ID
request_id = ''

# Create DRMS client, use debug=True to see the query URLs.
c = drms.Client(verbose=True)

# Ask for a RequestID, if it is not set yet.
if not request_id:
    request_id = input('Please enter a RequestID: ')
    print()

# Querying the server using the entered RequestID.
print('Looking up export request "%s"...' % request_id)
r = c.export_from_id(request_id)

# Print request URL and number of available files.
print('\nRequest URL: %s' % r.request_url)
print('%d file(s) available for download.\n' % len(r.urls))

# Ask if the files should be downloaded.
do_download = input('Retrieve all files [y/N]? ')
print()

if do_download.lower() in ['y', 'yes']:
    # Create download directory if it does not exist yet.
    out_dir = os.path.join('downloads', request_id)
    if not os.path.exists(out_dir):
        os.makedirs(out_dir)

    # Download all available files.
    r.download(out_dir)
    print('Download finished.')
    print('\nDownload directory:\n  "%s"\n' % os.path.abspath(out_dir))


text:
package com.cloudera.flume.handlers.avro;

import java.io.IOException;
import java.io.OutputStream;

import org.apache.avro.AvroRuntimeException;
import org.apache.avro.Schema;
import org.apache.avro.file.CodecFactory;
import org.apache.avro.file.DataFileWriter;
import org.apache.avro.io.DatumWriter;
import org.apache.avro.reflect.ReflectData;
import org.apache.avro.reflect.ReflectDatumWriter;

import com.cloudera.flume.core.Event;
import com.cloudera.flume.core.EventImpl;
import com.cloudera.flume.handlers.text.FormatFactory.OutputFormatBuilder;
import com.cloudera.flume.handlers.text.output.AbstractOutputFormat;
import com.cloudera.flume.handlers.text.output.OutputFormat;
import com.google.common.base.Preconditions;

/**
 * This uses the Avro's Reflection serializer and DataFileWriter to output Flume
 * events to a AvroDataFile. Under the hood this uses the BinaryEncoder for
 * efficient storage and also as another wrapper layer for partitioning blocks
 * that allow for splitting.
 *
 * This is not thread safe.
 */
public class AvroDataFileOutputFormat extends AbstractOutputFormat {

  final static ReflectData reflectData = ReflectData.get();
  final static Schema schema = reflectData.getSchema(EventImpl.class);

  private static final String NAME = "avrodata";

  DatumWriter<EventImpl> writer = new ReflectDatumWriter<EventImpl>(schema);

  DataFileWriter<EventImpl> sink = null;
  OutputStream cachedOut = null;
  CodecFactory codecFactory = null;

  public AvroDataFileOutputFormat() {
    this(null);
  }

  public AvroDataFileOutputFormat(CodecFactory codecFactory) {
    this.codecFactory = codecFactory;
  }

  @Override
  public void format(OutputStream o, Event e) throws IOException {
    if (sink == null) {
      // first time, no current OutputStream or sink.
      cachedOut = o;
      sink = new DataFileWriter<EventImpl>(writer);
      if (codecFactory != null) {
        try {
          sink.setCodec(codecFactory);
        } catch (AvroRuntimeException ex) {
          sink = null;
          cachedOut = null;
          throw new IOException("Error setting Avro datafile codec", ex);
        }
      }
      sink.create(schema, o); // this opens
    }

    if (cachedOut != o) {
      // different output than last time, fail here
      throw new IOException(
          "OutputFormat instance can only write to the same OutputStream");
    }

    EventImpl ei = null;
    if (e instanceof EventImpl) {
      ei = (EventImpl) e;
    } else {
      // copy constructor to force into an EventImpl for reflection
      ei = new EventImpl(e);
    }
    sink.append(ei);
    sink.flush();
  }

  public static OutputFormatBuilder builder() {
    return new OutputFormatBuilder() {

      @Override
      public OutputFormat build(String... args) {
        Preconditions.checkArgument(args.length <= 2, "usage: avrodata([codec, [compressionLevel]])");

        String codecName = null;
        int compressionLevel = -1;
        if (args.length > 0) {
          codecName = args[0];
          if (args.length > 1) {
            compressionLevel = Integer.parseInt(args[1]);
          }
        }

        CodecFactory codecFactory = null;
        try {
          if ((codecName != null) && codecName.equals("deflate") && (compressionLevel != -1)) {
            codecFactory = CodecFactory.deflateCodec(compressionLevel);
          } else if (codecName != null) {
            codecFactory = CodecFactory.fromString(codecName);
          }
        } catch (AvroRuntimeException ex) {
          throw new IllegalArgumentException("Invalid Avro data file codec", ex);
        }

        OutputFormat format = new AvroDataFileOutputFormat(codecFactory);
        format.setBuilder(this);

        return format;
      }

      @Override
      public String getName() {
        return NAME;
      }

    };
  }

}


text:
<?php

namespace Neodirectorio\PhenixBundle\Entity;

use Doctrine\ORM\Mapping as ORM;

/**
 * Album
 *
 * @ORM\Table()
 * @ORM\Entity
 */
class Album
{
    /**
     * @var integer
     *
     * @ORM\Column(name="id", type="integer")
     * @ORM\Id
     * @ORM\GeneratedValue(strategy="AUTO")
     */
    private $id;

    /**
     * @var string
     *
     * @ORM\Column(name="nombre", type="string", length=255)
     */
    private $nombre;

    /**
     * @var string
     *
     * @ORM\Column(name="descripcion", type="string", length=255)
     */
    private $descripcion;

    /**
     * @var \DateTime
     *
     * @ORM\Column(name="fechaCreacion", type="datetime")
     */
    private $fechaCreacion;

    /**
     * @ORM\OneToMany(targetEntity="Imagen", mappedBy="parent")
     */
    private $child;

    /**
     *
     * @ORM\ManyToOne(targetEntity="Participante", inversedBy="albumes")
     * @ORM\JoinColumn(name="participante_id", referencedColumnName="id")
     */
    private $creador;

    /**
     * @var integer
     *
     * @ORM\Column(name="cantFotos", type="integer")
     */
    private $cantFotos;

    /**
     * Get id
     *
     * @return integer
     */
    public function getId()
    {
        return $this->id;
    }

    /**
     * Set nombre
     *
     * @param string $nombre
     * @return Album
     */
    public function setNombre($nombre)
    {
        $this->nombre = $nombre;

        return $this;
    }

    /**
     * Get nombre
     *
     * @return string
     */
    public function getNombre()
    {
        return $this->nombre;
    }

    /**
     * Set descripcion
     *
     * @param string $descripcion
     * @return Album
     */
    public function setDescripcion($descripcion)
    {
        $this->descripcion = $descripcion;

        return $this;
    }

    /**
     * Get descripcion
     *
     * @return string
     */
    public function getDescripcion()
    {
        return $this->descripcion;
    }

    /**
     * Set fechaCreacion
     *
     * @param \DateTime $fechaCreacion
     * @return Album
     */
    public function setFechaCreacion($fechaCreacion)
    {
        $this->fechaCreacion = $fechaCreacion;

        return $this;
    }

    /**
     * Get fechaCreacion
     *
     * @return \DateTime
     */
    public function getFechaCreacion()
    {
        return $this->fechaCreacion;
    }
    /**
     * Constructor
     */
    public function __construct()
    {
        $this->child = new \Doctrine\Common\Collections\ArrayCollection();
        $this->cantFotos=0;
    }

    /**
     * Add child
     *
     * @param \Neodirectorio\PhenixBundle\Entity\Imagen $child
     * @return Album
     */
    public function addChild(\Neodirectorio\PhenixBundle\Entity\Imagen $child)
    {
        $this->child[] = $child;
        $this->cantFotos=$this->cantFotos+1;

        return $this;
    }

    /**
     * Remove child
     *
     * @param \Neodirectorio\PhenixBundle\Entity\Imagen $child
     */
    public function removeChild(\Neodirectorio\PhenixBundle\Entity\Imagen $child)
    {
        $this->child->removeElement($child);
        $this->cantFotos=$this->cantFotos-1;
    }

    /**
     * Get child
     *
     * @return \Doctrine\Common\Collections\Collection
     */
    public function getChild()
    {
        return $this->child;
    }

    /**
     * Set creador
     *
     * @param \Neodirectorio\PhenixBundle\Entity\Participante $creador
     * @return Album
     */
    public function setCreador(\Neodirectorio\PhenixBundle\Entity\Participante $creador = null)
    {
        $this->creador = $creador;

        return $this;
    }

    /**
     * Get creador
     *
     * @return \Neodirectorio\PhenixBundle\Entity\Participante
     */
    public function getCreador()
    {
        return $this->creador;
    }

    /**
     * Set cantFotos
     *
     * @param integer $cantFotos
     * @return Album
     */
    public function setCantFotos($cantFotos)
    {
        $this->cantFotos = $cantFotos;

        return $this;
    }

    /**
     * Get cantFotos
     *
     * @return integer
     */
    public function getCantFotos()
    {
        return $this->cantFotos;
    }

    /**
     * Get cantFotos
     *
     * @return integer
     */
    public function updateOrdinal($ord)
    {
        $imagen=new \Neodirectorio\PhenixBundle\Entity\Participante();
        $hijos=$this->child;
        for ($i=0;$i<$this->cantFotos;$i++)
        {
            $imagen=$hijos->get($i);
            if ($imagen->getOrdinal()>$ord)
            {
                $var=$imagen->getOrdinal();
                $imagen->setOrdinal($var-1);
            }
        }
    }
}


text:
<?php /** @noinspection SqlNoDataSourceInspection */
/** @noinspection SqlDialectInspection */

/*
 * This file is part of the Stinger Doctrine-Commons package.
 *
 * (c) Oliver Kotte <oliver.kotte@stinger-soft.net>
 * (c) Florian Meyer <florian.meyer@stinger-soft.net>
 *
 * For the full copyright and license information, please view the LICENSE
 * file that was distributed with this source code.
 */

namespace StingerSoft\DoctrineCommons\Utils;

use Doctrine\DBAL\Connection;
use Doctrine\DBAL\DBALException;
use PDO;
use function count;

/**
 * Exports all tables as json formatted data
 */
class JsonExporter implements ExporterInterface {

	public const CHUNK_SIZE = 300000;

	/**
	 *
	 * @var Connection
	 */
	protected $connection;

	/**
	 * @var callable[]
	 */
	protected $listeners = array();

	/**
	 * Default constructor
	 *
	 * @param Connection $connection
	 */
	public function __construct(Connection $connection) {
		$this->connection = $connection;
	}

	/**
	 *
	 * {@inheritdoc}
	 *
	 * @throws DBALException
	 * @see \StingerSoft\DoctrineCommons\Utils\ExporterInterface::exportToFilename()
	 */
	public function exportToFilename($filename): int {
		$handle = fopen($filename, 'wb');
		$rows = $this->export($handle);
		fclose($handle);
		return $rows;
	}

	public function addListener(callable $listener): void {
		$this->listeners[] = $listener;
	}

	protected function callListeners($currentTableName, $tableNum, $tableCount, $rowNum, $rowCount): void {
		foreach($this->listeners as $listener) {
			call_user_func($listener, $currentTableName, $tableNum, $tableCount, $rowNum, $rowCount);
		}
	}

	/**
	 *
	 * {@inheritdoc}
	 *
	 * @throws DBALException
	 * @see \StingerSoft\DoctrineCommons\Utils\ExporterInterface::export()
	 * @noinspection DisconnectedForeachInstructionInspection
	 */
	public function export($resource): int {
		$tables = $this->connection->getSchemaManager()->listTables();

		$i = 0;
		$len = count($tables);
		$totalRows = 0;

		fwrite($resource, '{');
		foreach($tables as $table) {

			$useHackForLargeTables = false;
			$primaryKeys = $table->hasPrimaryKey() ? $table->getPrimaryKeyColumns() : array();
			$lastPrimaryKeyValue = null;

			$countQuery = 'SELECT COUNT(*) as count FROM ' . $table->getName();
			$count = (int)current($this->connection->executeQuery($countQuery)->fetch(PDO::FETCH_ASSOC));
			$totalRows += $count;

			if($count > self::CHUNK_SIZE * 10 && count($primaryKeys) === 1) {
				$useHackForLargeTables = true;
			}

			$qb = $this->connection->createQueryBuilder();
			$qb->select('*');
			$qb->from($table->getName());
			$qb->setMaxResults(self::CHUNK_SIZE);
			if($useHackForLargeTables) {
				$qb->orderBy($primaryKeys[0]);
			}

			fwrite($resource, '"' . $table->getName() . '":');
			$delim = '';

			fwrite($resource, '[');
			$pages = ceil($count / self::CHUNK_SIZE);

			for($page = 0; $page < $pages; $page++) {
				$this->callListeners($table->getName(), $i, $len, $page * self::CHUNK_SIZE, $count);
				if($useHackForLargeTables) {
					if($lastPrimaryKeyValue !== null) {
						$qb->where($primaryKeys[0] . ' > ' . $lastPrimaryKeyValue);
					}
				} else {
					$qb->setFirstResult($page * self::CHUNK_SIZE);
				}

				$stmt = $qb->execute();
				while(($row = $stmt->fetch(PDO::FETCH_ASSOC)) !== false) {
					if(isset($row['doctrine_rownum'])) {
						unset($row['doctrine_rownum']);
					}
					fwrite($resource, $delim);
					fwrite($resource, json_encode($row));
					$delim = ',';
					if($useHackForLargeTables) {
						$lastPrimaryKeyValue = $row[$primaryKeys[0]];
					}
				}
			}
			fwrite($resource, ']');
			if($count !== 0) {
				$this->callListeners($table->getName(), $i, $len, $count, $count);
			}

			// if not last table
			if($i != $len - 1) {
				fwrite($resource, ',');
			}
			$i++;
		}

		fwrite($resource, '}');
		return $totalRows;
	}
}