text:
The TestGrid API controller exposes TestGrid data publicly that could otherwise be viewed through the UI.

### Usage

The surface of the API is described in this (proto definition)[/pb/api/v1/data.proto]. Usage is similar between protocols.

You may want to set `--scope=gs://your-bucket`. This will set this as the server's default; otherwise you'll be required to specify with each call.

#### HTTP

Use the `--http-port` option to set the listening port. Default is 8080.

You can specify further with URL parameters: `?scope=gs://your-bucket`.

`curl localhost:8080/api/v1/dashboards`

#### gRPC

Use the `--grpc-port` option to set the listening port. Default is 50051.

`grpc_cli call localhost:50051 testgrid.api.v1.TestGridData.ListDashboard`

Reflection is enabled in gRPC, allowing you to ask the server what methods are available.

`grpc_cli ls localhost:50051 testgrid.api.v1.TestGridData`


text:
from test_framework.authproxy import JSONRPCException
from test_framework.test_framework import ElysiumTestFramework
from test_framework.util import assert_equal, assert_raises_message

class ElysiumIssuanceManagedTest(ElysiumTestFramework):
    def run_test(self):
        super().run_test()

        # check parameter value validation
        assert_raises_message(
            JSONRPCException,
            'Invalid address',
            self.nodes[0].elysium_sendissuancemanaged, 'abc', 1, 1, 0, 'category1', 'subcategory1', 'token1', 'http://foo.com', 'data1'
        )

        assert_raises_message(
            JSONRPCException,
            'Invalid ecosystem (1 = main, 2 = test only)',
            self.nodes[0].elysium_sendissuancemanaged, self.addrs[0], 0, 1, 0, 'category1', 'subcategory1', 'token1', 'http://foo.com', 'data1'
        )

        assert_raises_message(
            JSONRPCException,
            'Invalid ecosystem (1 = main, 2 = test only)',
            self.nodes[0].elysium_sendissuancemanaged, self.addrs[0], 3, 1, 0, 'category1', 'subcategory1', 'token1', 'http://foo.com', 'data1'
        )

        assert_raises_message(
            JSONRPCException,
            'Invalid property type (1 = indivisible, 2 = divisible only)',
            self.nodes[0].elysium_sendissuancemanaged, self.addrs[0], 1, 0, 0, 'category1', 'subcategory1', 'token1', 'http://foo.com', 'data1'
        )

        assert_raises_message(
            JSONRPCException,
            'Invalid property type (1 = indivisible, 2 = divisible only)',
            self.nodes[0].elysium_sendissuancemanaged, self.addrs[0], 1, 3, 0, 'category1', 'subcategory1', 'token1', 'http://foo.com', 'data1'
        )

        assert_raises_message(
            JSONRPCException,
            'Property appends/replaces are not yet supported',
            self.nodes[0].elysium_sendissuancemanaged, self.addrs[0], 1, 1, 1, 'category1', 'subcategory1', 'token1', 'http://foo.com', 'data1'
        )

        assert_raises_message(
            JSONRPCException,
            'Text must not be longer than 255 characters',
            self.nodes[0].elysium_sendissuancemanaged, self.addrs[0], 1, 1, 0, 'c' * 256, 'subcategory1', 'token1', 'http://foo.com', 'data1'
        )

        assert_raises_message(
            JSONRPCException,
            'Text must not be longer than 255 characters',
            self.nodes[0].elysium_sendissuancemanaged, self.addrs[0], 1, 1, 0, 'category1', 's' * 256, 'token1', 'http://foo.com', 'data1'
        )

        assert_raises_message(
            JSONRPCException,
            'Text must not be longer than 255 characters',
            self.nodes[0].elysium_sendissuancemanaged, self.addrs[0], 1, 1, 0, 'category1', 'subcategory1', 't' * 256, 'http://foo.com', 'data1'
        )

        assert_raises_message(
            JSONRPCException,
            'Text must not be longer than 255 characters',
            self.nodes[0].elysium_sendissuancemanaged, self.addrs[0], 1, 1, 0, 'category1', 'subcategory1', 'token1', 'h' * 256, 'data1'
        )

        assert_raises_message(
            JSONRPCException,
            'Text must not be longer than 255 characters',
            self.nodes[0].elysium_sendissuancemanaged, self.addrs[0], 1, 1, 0, 'category1', 'subcategory1', 'token1', 'http://foo.com', 'd' * 256
        )

        assert_raises_message(
            JSONRPCException,
            'Property name must not be empty',
            self.nodes[0].elysium_sendissuancemanaged, self.addrs[0], 1, 1, 0, 'category1', 'subcategory1', '', 'http://foo.com', 'data1'
        )

        assert_raises_message(
            JSONRPCException,
            'Sigma status is not valid',
            self.nodes[0].elysium_sendissuancemanaged, self.addrs[0], 1, 1, 0, 'category1', 'subcategory1', 'token1', 'http://foo.com', 'data1', 4
        )

        assert_raises_message(
            JSONRPCException,
            'Sigma feature is not activated yet',
            self.nodes[0].elysium_sendissuancemanaged, self.addrs[0], 1, 1, 0, 'category1', 'subcategory1', 'token1', 'http://foo.com', 'data1', 1
        )

        # create properties
        tx1 = self.nodes[0].elysium_sendissuancemanaged(self.addrs[0], 1, 1, 0, 'main', 'indivisible', 'token1', 'http://token1.com', 'data1')
        self.nodes[0].generate(150) # we need 100 blocks in order to specify sigma flag
        self.sync_all()

        tx2 = self.nodes[1].elysium_sendissuancemanaged(self.addrs[1], 1, 2, 0, 'main', 'divisible', 'token2', 'http://token2.com', 'data2', 0)
        self.nodes[1].generate(1)
        self.sync_all()

        tx3 = self.nodes[2].elysium_sendissuancemanaged(self.addrs[2], 2, 1, 0, 'test', 'indivisible', 'token3', 'http://token3.com', 'data3', 1)
        self.nodes[2].generate(1)
        self.sync_all()

        tx4 = self.nodes[3].elysium_sendissuancemanaged(self.addrs[3], 2, 2, 0, 'test', 'divisible', 'token4', 'http://token4.com', 'data4', 2)
        self.nodes[3].generate(1)
        self.sync_all()

        tx5 = self.nodes[0].elysium_sendissuancemanaged(self.addrs[0], 1, 1, 0, 'main', 'indivisible', 'token5', 'http://token5.com', 'data5', 3)
        self.nodes[0].generate(1)
        self.sync_all()

        # check property creation
        props = self.nodes[0].elysium_listproperties()

        assert_equal(len(props), 2 + 5) # 2 pre-defined properties + 5 new created

        self.assert_property_summary(props[2], 3, False, 'main', 'indivisible', 'token1', 'http://token1.com', 'data1')
        self.assert_property_summary(props[3], 4, True, 'main', 'divisible', 'token2', 'http://token2.com', 'data2')
        self.assert_property_summary(props[4], 5, False, 'main', 'indivisible', 'token5', 'http://token5.com', 'data5') # main eco tokens will come first
        self.assert_property_summary(props[5], 2147483651, False, 'test', 'indivisible', 'token3', 'http://token3.com', 'data3')
        self.assert_property_summary(props[6], 2147483652, True, 'test', 'divisible', 'token4', 'http://token4.com', 'data4')

        self.assert_property_info(
            self.nodes[1].elysium_getproperty(3),
            3,
            False,
            self.addrs[0],
            False,
            'main',
            'indivisible',
            'token1',
            'http://token1.com',
            'data1',
            '0',
            'SoftDisabled',
            tx1,
            [])
        self.assert_property_info(
            self.nodes[2].elysium_getproperty(4),
            4,
            False,
            self.addrs[1],
            True,
            'main',
            'divisible',
            'token2',
            'http://token2.com',
            'data2',
            '0.00000000',
            'SoftDisabled',
            tx2,
            [])
        self.assert_property_info(
            self.nodes[1].elysium_getproperty(5),
            5,
            False,
            self.addrs[0],
            False,
            'main',
            'indivisible',
            'token5',
            'http://token5.com',
            'data5',
            '0',
            'HardEnabled',
            tx5,
            [])
        self.assert_property_info(
            self.nodes[3].elysium_getproperty(2147483651),
            2147483651,
            False,
            self.addrs[2],
            False,
            'test',
            'indivisible',
            'token3',
            'http://token3.com',
            'data3',
            '0',
            'SoftEnabled',
            tx3,
            [])
        self.assert_property_info(
            self.nodes[0].elysium_getproperty(2147483652),
            2147483652,
            False,
            self.addrs[3],
            True,
            'test',
            'divisible',
            'token4',
            'http://token4.com',
            'data4',
            '0.00000000',
            'HardDisabled',
            tx4,
            [])

if __name__ == '__main__':
    ElysiumIssuanceManagedTest().main()

redpajama_set_name: RedPajamaGithub


text:
<?xml version="1.0" encoding="utf-8"?>
<shape xmlns:android="http://schemas.android.com/apk/res/android">
    <solid android:color="@color/color_25"/>
    <corners android:radius="30dp"/>
    <stroke
        android:width="1dp"
        android:color="#EBEBEB"/>
    <padding
        android:bottom="4dp"
        android:left="11dp"
        android:right="11dp"
        android:top="4dp"/>
</shape>


text:
from schedule import wrappers

def get_scheduled_tag_value(instance):
    try:
        retval = [t['Value'] for t in instance['Tags'] if t['Key'] == 'Scheduled'][0]
    except IndexError:
        retval = ''
    return retval

def is_running(instance):
    return instance.state == AWS_INSTANCE_STATES.RUNNING

def is_stopped(instance):
    return instance.state == AWS_INSTANCE_STATES.STOPPED

def running_ids(instances):
    return [instance for instance in instances if is_running(instance)]

def stopped_ids(instances):
    return [instance for instance in instances if is_stopped(instance)]

# aws instance states
class AWS_INSTANCE_STATES:
    PENDING = [0, 'pending']
    RUNNING = [16, 'running']
    SHUTTING = [32, 'shutting-down']
    TERMINATED = [48, 'terminated']
    STOPPING = [64, 'stopping']
    STOPPED = [80, 'stopped']

def create_aws_instance_if_scheduled(instance):
    """Given the json version of an instance, create an AWSScheduledInstance wrapper"""
    schedule = get_scheduled_tag_value(instance)
    if not schedule:
        retval = None
    else:
        retval = wrappers.AWSScheduledInstance(instance, schedule)
    return retval


text:
<manifest xmlns:android="http://schemas.android.com/apk/res/android" android:versionCode="1" android:versionName="1.0" package="ca.mlht.android.skip">

    <uses-sdk android:minSdkVersion="9" android:targetSdkVersion="16"/>

    <application android:allowBackup="true" android:icon="@drawable/ic_launcher" android:label="@string/app_name" android:theme="@style/AppTheme">
        <activity android:label="@string/app_name" android:name="ca.mlht.android.skip.MainActivity">
            <intent-filter>
                <action android:name="android.intent.action.MAIN"/>

                <category android:name="android.intent.category.LAUNCHER"/>
            </intent-filter>
        </activity>
        <receiver android:name="ca.mlht.android.skip.NotificationReceiver" android:permission="com.google.android.c2dm.permission.SEND">
            <intent-filter>
                <action android:name="com.google.android.c2dm.intent.RECEIVE"/>
                <category android:name="ca.mlht.android.skip"/>
            </intent-filter>
        </receiver>
    </application>
    <uses-permission android:name="android.permission.INTERNET"/>
    <uses-permission android:name="android.permission.GET_ACCOUNTS"/>
    <uses-permission android:name="com.google.android.c2dm.permission.RECEIVE"/>
    <permission android:name="ca.mlht.android.skip.permission.C2D_MESSAGE" android:protectionLevel="signature"/>
    <uses-permission android:name="ca.mlht.android.skip.android.permission.C2D_MESSAGE"/>
</manifest>


text:
import os

from distutils.core import setup

setup(
    name='fSequenceTools',
    version='1.0.0',
    description='Sequence numeric encoding and report building tools',
    long_description=\
    '''
==============
fSequenceTools
==============

This package was created to investigate two separate aspects of sequences
analysis:

Encoding time
    By using threading the reading job is split between reading data,
    detecting formats, and encoding sequence data. The structure is fully
    extensible to cover more formats and new types of encoders.

Reporting
    Reports are put together in groups so that secondary analysis of
    encoded data can be shared by a group of different ploting and statistics
    producers.

    ''',
    author='Martin Zackrisson',
    author_email='martin.zackrisson@gu.se',
    url='https://gitorious.org/fseq',
    packages=['fseq', 'fseq.reading', 'fseq.reporting'],
    licence='MIT',
    scripts=[os.path.join("scripts", p) for p in ("fseq",)],
    requires=['numpy', 'scipy', 'matplotlib'],
    classifiers=[
        'Development Status :: 3 - Alpha',
        'Intended Audience :: Science/Research',
        'License :: OSI Approved :: MIT License',
        'Natural Language :: English',
        'Operating System :: POSIX',
        'Operating System :: MacOS',
        'Operating System :: Microsoft',
        'Programming Language :: Python :: 2.7',
        'Programming Language :: Python :: 3.4',
        'Topic :: Scientific/Engineering :: Bio-Informatics',
    ]
)


text:
Squirrel Notes is a simple notes application, inspired by [National Velocity](http://notational.net) and [nvALT](http://brettterpstra.com/projects/nvalt/). Its purpose is to allow the creation of rich HTML rendered notes, while allowing lightning fast retrieval using tags and fuzzy searching.

![application image](app.png)

## Motivation
I love the simplicity and `stay out of your way` style of NV and nvALT, but their lack of built in HTML rendering lets them down. Also this was the perfect opportunity to take [Electron](https://github.com/atom/electron) for a spin.

## Dependencies
- [NPM 3](http://blog.npmjs.org/post/122450408965/npm-weekly-20-npm-3-is-here-ish) (`npm install -g npm@3.0-latest`)
- [Gulp](https://www.npmjs.com/package/gulp) (globally installed)
- [Babel](https://www.npmjs.com/package/babel) (globally installed)

## Development
To set up a development environment, simply do:
```sh
git clone git@github.com:christian-schulze/SquirrelNotes.git
cd squirrelnotes
npm install --dev
gulp build
```
Then you can:
```sh
gulp build # build everything into the app folder
gulp watch # rebuild when something changes
gulp package # Create a stand-alone application in the packages folder
npm start # start the app in development mode
```

## TODO
- [x] Add note
- [x] Rename note (change title)
- [x] Delete note
- [ ] Optimize for keyboard-only use (mouse should be completely optional)
- [ ] Allow vertical resizing of results table (and persist the size)
- [ ] Rework html/css layout, so resizing works better
- [ ] Implement better editor ([AlloyEditor](http://alloyeditor.com))
- [ ] Add auto-updater (when new build is available)
- [ ] Refactor using the flux pattern (specifically [Redux](https://github.com/rackt/redux))
- [ ] Add tests ([electron-mocka](https://github.com/jprichardson/electron-mocha) and/or [selenium/webdriver](http://electron.atom.io/docs/v0.31.0/tutorial/using-selenium-and-webdriver/))

## License
[MIT](./LICENSE)


text:
package org.xlcloud.rest.client.cache;

import java.io.IOException;
import java.util.ArrayList;
import java.util.Date;
import java.util.Iterator;
import java.util.List;

import javax.annotation.PostConstruct;
import javax.cache.Cache;
import javax.cache.Cache.Entry;
import javax.cache.CacheBuilder;
import javax.cache.CacheManager;
import javax.cache.Caching;
import javax.enterprise.context.ApplicationScoped;
import javax.inject.Inject;

import org.apache.commons.lang.StringUtils;
import org.xlcloud.config.ConfigParam;
import org.xlcloud.logging.LoggingUtils;

import com.fasterxml.jackson.databind.ObjectMapper;
import com.sun.jersey.api.client.ClientResponse;
import com.sun.jersey.api.client.WebResource;
import com.sun.research.ws.wadl.HTTPMethods;

/**
 *  Implementation of cache based on Jcache
 *
 * @author Michał Kamiński, AMG.net
 */
@ApplicationScoped
public class EntitiesCache {

    private Cache<CacheKey, CacheObject> responsesCache;

    @Inject
    ObjectMapper mapper;

    @Inject
    @ConfigParam
    String xmsApiUri;

    public EntitiesCache() {
    }

    @PostConstruct
    public void init() {
        CacheManager manager = Caching.getCacheManager("xms-cache");
        CacheBuilder<CacheKey, CacheObject> cacheBuilder = manager.createCacheBuilder("xmsObjectsCache");
        responsesCache = cacheBuilder.build();
    }

    private static final org.apache.log4j.Logger LOG = org.apache.log4j.Logger.getLogger(EntitiesCache.class);

    /**
     * Caches response if method is different than POST and PUT and response
     * status is correct. Caching policy of entity is based on Cache-Control
     * header of response
     *
     * @param request
     * @param authorization
     * @param response
     */
    public <T> T cacheResponse(WebResource request, String authorization, ClientResponse response, Class<T> c, String method) {
        T entity = response.getEntity(c);

        byte[] entityAsBytes;
        try {
            entityAsBytes = mapper.writer().writeValueAsBytes(entity);
        } catch (IOException e) {
            LOG.error(method + " " + request.getURI() + " has problem with serialization of entity", e);
            throw new RuntimeException(e.getMessage(), e);
        }

        if (!StringUtils.equals(method, HTTPMethods.PUT.name()) && !StringUtils.equals(method, HTTPMethods.POST.name())
                && response.getStatus() < 300 && response.getStatus() >= 200) {
            List<String> cacheContol = response.getHeaders().get("Cache-Control");
            if (cacheContol == null) {
                return entity;
            }
            boolean noCache = false;
            Long cacheTimeout = 0l;
            for (String cacheControlParam : cacheContol) {
                if (StringUtils.equals(cacheControlParam, "no-cache")) {
                    noCache = true;
                    break;
                } else if (StringUtils.contains(cacheControlParam, "max-age: ")) {
                    cacheTimeout = Long.parseLong(StringUtils.substringAfter(cacheControlParam, "max-age: "));
                }
            }

            if (noCache || cacheTimeout == null || cacheTimeout <= 0) {
                return entity;
            }
            responsesCache.put(new CacheKey(getPath(request), authorization, method),
                    new CacheObject(entityAsBytes, new Date(System.currentTimeMillis() + 1000l * cacheTimeout)));
        }
        return entity;
    }

    private String getPath(WebResource webResource) {
        return StringUtils.substringAfter(webResource.getURI().toString(), xmsApiUri);
    }

    /**
     * Retrieves from private cache entity and validates if cache entity does
     * not expired.
     *
     * @param webResource
     * @param authorization
     * @return
     */
    public <T> T retrieveFromCache(WebResource webResource, String authorization, Class<T> c, String method) {
        CacheObject cacheObject = responsesCache.get(new CacheKey(getPath(webResource), authorization, method));
        if (cacheObject != null) {
            if (!cacheObject.isObjectValid()) {
                responsesCache.remove(new CacheKey(getPath(webResource), authorization, method));
                return null;
            }
            try {
                return mapper.readValue(cacheObject.getCacheObjectBytes(), c);
            } catch (IOException e) {
                LOG.error(method + " " + LoggingUtils.maskResource(webResource.getURI().toASCIIString()) + " has problem with serialization of entity");
                throw new RuntimeException(e.getMessage(), e);
            }
        }
        return null;
    }

    /**
     * Removes from all public caches all entities which match
     * {@link ResourceComparator}
     *
     * @param resourceComparator
     */
    public void removeFromCache(ResourceComparator resourceComparator) {
        List<CacheKey> entriesToRemove = new ArrayList<>();
        Iterator<Entry<CacheKey, CacheObject>> entitiesIterator = responsesCache.iterator();
        while (entitiesIterator.hasNext()) {
            Entry<CacheKey, CacheObject> entry = entitiesIterator.next();
            if (resourceComparator.match(entry.getKey().getPath())) {
                entriesToRemove.add(entry.getKey());
            }
        }
        for (CacheKey cacheKey : entriesToRemove) {
            LOG.debug("Removing from cace: " + cacheKey);
            responsesCache.remove(cacheKey);
        }
    }
}


text:
using content::RenderViewHost;

WebUITestHandler::WebUITestHandler()
    : test_done_(false),
      test_succeeded_(false),
      run_test_done_(false),
      run_test_succeeded_(false),
      is_waiting_(false) {
}

void WebUITestHandler::PreloadJavaScript(const string16& js_text,
                                         RenderViewHost* preload_host) {
  DCHECK(preload_host);
  preload_host->Send(new ChromeViewMsg_WebUIJavaScript(
      preload_host->GetRoutingID(), string16(), js_text, 0,
      false));
}

void WebUITestHandler::RunJavaScript(const string16& js_text) {
  web_ui()->GetWebContents()->GetRenderViewHost()->ExecuteJavascriptInWebFrame(
      string16(), js_text);
}

bool WebUITestHandler::RunJavaScriptTestWithResult(const string16& js_text) {
  test_succeeded_ = false;
  run_test_succeeded_ = false;
  RenderViewHost* rvh = web_ui()->GetWebContents()->GetRenderViewHost();
  content::NotificationRegistrar notification_registrar;
  notification_registrar.Add(
      this, content::NOTIFICATION_EXECUTE_JAVASCRIPT_RESULT,
      content::Source<RenderViewHost>(rvh));
  rvh->ExecuteJavascriptInWebFrameNotifyResult(string16(), js_text);
  return WaitForResult();
}

void WebUITestHandler::RegisterMessages() {
  web_ui()->RegisterMessageCallback("testResult",
      base::Bind(&WebUITestHandler::HandleTestResult, base::Unretained(this)));
}

void WebUITestHandler::HandleTestResult(const ListValue* test_result) {
  // Quit the message loop if |is_waiting_| so waiting process can get result or
  // error. To ensure this gets done, do this before ASSERT* calls.
  if (is_waiting_)
    MessageLoopForUI::current()->Quit();

  SCOPED_TRACE("WebUITestHandler::HandleTestResult");

  EXPECT_FALSE(test_done_);
  test_done_ = true;
  test_succeeded_ = false;

  ASSERT_TRUE(test_result->GetBoolean(0, &test_succeeded_));
  if (!test_succeeded_) {
    std::string message;
    ASSERT_TRUE(test_result->GetString(1, &message));
    LOG(ERROR) << message;
  }
}

void WebUITestHandler::Observe(int type,
                               const content::NotificationSource& source,
                               const content::NotificationDetails& details) {
  // Quit the message loop if |is_waiting_| so waiting process can get result or
  // error. To ensure this gets done, do this before ASSERT* calls.
  if (is_waiting_)
    MessageLoopForUI::current()->Quit();

  ASSERT_EQ(content::NOTIFICATION_EXECUTE_JAVASCRIPT_RESULT, type);

  SCOPED_TRACE("WebUITestHandler::Observe");

  EXPECT_FALSE(run_test_done_);
  run_test_done_ = true;
  run_test_succeeded_ = false;

  Value* value = content::Details<std::pair<int, Value*> >(details)->second;
  ASSERT_TRUE(value->GetAsBoolean(&run_test_succeeded_));
}

bool WebUITestHandler::WaitForResult() {
  SCOPED_TRACE("WebUITestHandler::WaitForResult");
  test_done_ = false;
  run_test_done_ = false;
  is_waiting_ = true;

  // Either sync test completion or the testDone() will cause message loop
  // to quit.
  content::RunMessageLoop();

  // Run a second message loop when not |run_test_done_| so that the sync test
  // completes, or |run_test_succeeded_| but not |test_done_| so async tests
  // complete.
  if (!run_test_done_ || (run_test_succeeded_ && !test_done_)) {
    content::RunMessageLoop();
  }

  is_waiting_ = false;

  // To succeed the test must execute as well as pass the test.
  return run_test_succeeded_ && test_succeeded_;
}


text:
{
  val dirContents = dbutils.fs.ls("dbfs:/tmp/pagecounts")
  val size = dirContents.map(_.size).head / (1024 * 1024)
  println(s"$size megabytes\n")
}

// COMMAND ----------

val path = dbutils.fs.ls("dbfs:/tmp/pagecounts").sortWith { (a, b) => a.name > b.name }.head.path

// COMMAND ----------

// MAGIC %md
// MAGIC ### RDDs
// MAGIC
// MAGIC RDDs can be created by using the Spark Context object's `textFile()` method.

// COMMAND ----------

// MAGIC %md Create an RDD from the recent pagecounts file:

// COMMAND ----------

// Notice that this returns a RDD of Strings
val pagecountsRDD = sc.textFile(path)

// COMMAND ----------

// MAGIC %md There's one partition, because a gzipped file cannot be uncompressed in parallel.

// COMMAND ----------

pagecountsRDD.partitions.length

// COMMAND ----------

// MAGIC %md Let's increase the parallelism by repartitioning. I'm using 6 partitions, which is twice the number of available threads on Databricks Community Edition.

// COMMAND ----------

val pagecountsRDD2 = pagecountsRDD.repartition(6)
pagecountsRDD2.partitions.length

// COMMAND ----------

// MAGIC %md The `count` action counts how many items (lines) total are in the RDD (this requires a full scan of the file):

// COMMAND ----------

val fmt = new java.text.DecimalFormat
println(fmt.format(pagecountsRDD2.count()))

// COMMAND ----------

// MAGIC %md So the count shows that there are several million lines in the file. Notice that the `count()` action took some time to run because it had to read the entire file remotely from S3.

// COMMAND ----------

// MAGIC %md Let's take a look at some of the data. This will return faster, because `take()` doesn't have to process the whole file.

// COMMAND ----------

pagecountsRDD2.take(10).foreach(println)

// COMMAND ----------

// MAGIC %md Notice that each line in the file actually contains 2 strings and 2 numbers, but our RDD is treating each line as a long string. We'll fix this typing issue shortly by using a custom parsing function.

// COMMAND ----------

// MAGIC %md In the output above, the first column (like `aa`) is the Wikimedia project name. The following abbreviations are used for the first column:
// MAGIC
// MAGIC * wikibooks: `".b"`
// MAGIC * wiktionary: `".d"`
// MAGIC * wikimedia: `".m"`
// MAGIC * wikipedia mobile: `".mw"`
// MAGIC * wikinews: `".n"`
// MAGIC * wikiquote: `".q"`
// MAGIC * wikisource: `".s"`
// MAGIC * wikiversity: `".v"`
// MAGIC * mediawiki: `".w"`
// MAGIC
// MAGIC Projects without a period and a following character are Wikipedia projects. So, any line starting with the column `aa` refers to the Aragones language Wikipedia. Similarly, any line starting with the column `en` refers to the English language Wikipedia. `en.b` refers to English Language Wikibooks.
// MAGIC
// MAGIC The second column is the title of the page retrieved, the third column is the number of requests, and the fourth column is the size of the content returned.

// COMMAND ----------

// MAGIC %md Let's sum up the request counts per page in the English Wikipiedia, then pull back the top 10. This is a variation of the code on Slide 4.
// MAGIC

// COMMAND ----------

pagecountsRDD2.flatMap { line =>
  line.split("""\s+""") match {
    case Array(project, page, numRequests, contentSize) => Some((project, page, numRequests.toLong))
    case _ => None
  }
}.
filter { case (project, page, numRequests) => project == "en" }.
map { case (project, page, numRequests) => (page, numRequests) }.
reduceByKey(_ + _).
sortBy({ case (page, numRequests) => numRequests }, ascending = false).
take(100).
foreach { case (page, totalRequests) => println(s"$page: $totalRequests") }

// COMMAND ----------

// MAGIC %md Let's remove the special pages. Page like "Talk:Topic", "User:username", "Special:Something", and anything starting with a "." are just cluttering things up. Here's a modification of the above, with a slightly different `filter()` call.

// COMMAND ----------

val SkipPages = Array(
  """^Special:""".r,
  """^File:""".r,
  """^Category:""".r,
  """^User:""".r,
  """^Talk:""".r,
  """^Template:""".r,
  """^Help:""".r,
  """^Wikipedia:""".r,
  """^MediaWiki:""".r,
  """^Portal:""".r,
  """^Book:""".r,
  """^Draft:""".r,
  """^Education_Program:""".r,
  """^TimedText:""".r,
  """^Module:""".r,
  """^Topic:""".r,
  """^Images/""".r,
  """^%22//upload.wikimedia.org""".r,
  """^%22//en.wikipedia.org""".r
)

def isSpecialPage(pageTitle: String): Boolean = SkipPages.exists { r => r.findFirstIn(pageTitle).isDefined }

// COMMAND ----------

def keepPage(page: String) = (! page.startsWith(".")) && (! isSpecialPage(page))
val pagecountsRDD3 =
  pagecountsRDD2.flatMap { line =>
  line.split("""\s+""") match {
    case Array(project, page, numRequests, contentSize) if (project == "en") && keepPage(page) => Some((page, numRequests.toLong))
    case _ => None
  }
}.
reduceByKey(_ + _).
sortBy({ case (page, numRequests) => numRequests }, ascending = false)

pagecountsRDD3.take(100).foreach { case (page, totalRequests) => println(s"$page: $totalRequests") }

// COMMAND ----------

// MAGIC %md That's completely type-safe, but it's up to us to choose the right implementation. For instance, the code, above, _could_ have by done like this:
// MAGIC
// MAGIC ```
// MAGIC pagecountsParsedRDD2.flatMap { ... }.
// MAGIC                      filter { ... }.
// MAGIC                      map { case (project, page, numRequests) => (page, numRequests) }.
// MAGIC                      groupByKey(_._1).
// MAGIC                      reduce(_ + _).
// MAGIC                      ...
// MAGIC ```
// MAGIC
// MAGIC However, `groupByKey() + reduce()` is _far_ more inefficient than `reduceByKey()`. Yet, Spark cannot protect us from choosing the wrong transformations.

// COMMAND ----------

// MAGIC %md Before we move on to DataFrames, let's cache this last RDD and see how much memory it uses.

// COMMAND ----------

pagecountsRDD3.setName("pagecountsRDD3").cache()
val totalPagesRDD = pagecountsRDD3.count() // we need to run an action to fill the cache
println(fmt.format(totalPagesRDD))

// COMMAND ----------

// MAGIC %md
// MAGIC ## DataFrames
// MAGIC
// MAGIC Let's try the same thing with DataFrames.
// MAGIC
// MAGIC To make a DataFrame, we can simply our RDD into another RDD of a different type, something Spark can use to infer a _schema_. Since we have fewer than 23 columns, we'll use a case class.

// COMMAND ----------

object Parser extends Serializable { // This helps with scoping issues in the notebook
  case class EditEntry(project: String, pageTitle: String, numberOfRequests: Long)

  def parseLine(line: String) = {
    line.split("""\s+""") match {
      case Array(project, page, numRequests, _) =>
        Some(EditEntry(project, page, numRequests.toLong))
      case _ =>
        None
    }
  }
}
val pagecountsDF = pagecountsRDD2.flatMap(Parser.parseLine).toDF

// COMMAND ----------

pagecountsDF.printSchema()

// COMMAND ----------

pagecountsDF.rdd.partitions.length

// COMMAND ----------

// MAGIC %md Of course, it would be more efficient to read the DataFrame from something other than a gzipped text file. Let's try that, as well. We'll save it to a Parquet file and read it back.

// COMMAND ----------

import org.apache.spark.sql.SaveMode
pagecountsDF.write.mode(SaveMode.Overwrite).parquet("dbfs:/tmp/pagecounts.parquet")

// COMMAND ----------

// MAGIC %fs ls /tmp/pagecounts.parquet

// COMMAND ----------

// Note the use of "spark", not "sqlContext". "spark" is a preinstantiated
// SparkSession, introduced in 2.0.
val pagecountsDFParquet = spark.read.parquet("dbfs:/tmp/pagecounts.parquet")

// COMMAND ----------

pagecountsDFParquet.rdd.partitions.length

// COMMAND ----------

// MAGIC %md Let's get rid of the special pages, as we did with the RDD version.

// COMMAND ----------

import org.apache.spark.sql.functions._

val uKeepPage = sqlContext.udf.register("keepPage", keepPage _)

val pagecountsDF2 = pagecountsDF.filter($"project" === "en").
                                 filter(uKeepPage($"pageTitle")).
                                 filter(substring($"pageTitle", 0, 1) !== ".").
                                 groupBy($"pageTitle").
                                 agg(sum($"numberOfRequests").as("count")).
                                 orderBy($"count".desc)
pagecountsDF2.take(100).foreach { row =>
  println(s"${row(0)}: ${row(1)}")
}

// COMMAND ----------

// MAGIC %md Easier to read, but... not type-safe.

// COMMAND ----------

// MAGIC %md
// MAGIC ### A very brief aside, about partitions
// MAGIC
// MAGIC According to Spark documentation, when a DataFrame shuffle occurs, the number of post-shuffle partitions is defined by Spark configuration parameter
// MAGIC `spark.sql.shuffle.partitions`, which defaults to 200. So, we should have that many partitions in `pagecountsDF2`, because the `groupBy` and `agg` calls, above, are likely to produce shuffles.
// MAGIC
// MAGIC But...

// COMMAND ----------

println(s"spark.sql.shuffle.partitions = ${sqlContext.getConf("spark.sql.shuffle.partitions")}")
println(s"pagecountsDF2 partitions     = ${pagecountsDF2.rdd.partitions.length}")

// COMMAND ----------

// MAGIC %md Okay, that's not 200.
// MAGIC
// MAGIC If we reran the creation of `pagecountsDF2`, above, we might see 18, 19, 20, 21, 22, or some number in that area for the number of post-shuffle partitions. Why?
// MAGIC
// MAGIC It turns out that the `orderBy`, above, uses _range partitioning_. To determine reasonable upper and lower bounds for the range, Spark randomly samples the data. In this case, we end up with something around 22 partitions; the partition could might differ with different data. It's not always the same from run to run because the random sampling doesn't use the same seed every time.

// COMMAND ----------

// MAGIC %md
// MAGIC ### Back to our DataFrame
// MAGIC
// MAGIC The `pagecountsDF2` DataFrame consists of `Row` objects, which can contain heterogenous types. If we pull some `Rows` back to the driver, we can extract the columns, but only as type `Any`:

// COMMAND ----------

val first10Rows = pagecountsDF2.take(10)

// COMMAND ----------

val row = first10Rows.head
row(0)

// COMMAND ----------

// MAGIC %md Note that `Row` isn't typed. It can't be, since each row consists of columns of potentially different types. (I suppose Spark could've used Shapeless, but it didn't...) If we want to get back to actual, useful Scala types, we have to do something ugly, like this:

// COMMAND ----------

first10Rows.map { row =>
  (row(0).asInstanceOf[String], row(1).asInstanceOf[Long])
}

// COMMAND ----------

// MAGIC %md Before we move on to Datasets, let's:
// MAGIC * verify that the number of items in the DataFrame match the RDD with the special pages filtered out
// MAGIC * cache the DataFrame
// MAGIC * compare the cached size to the cached size of the RDD.

// COMMAND ----------

val totalPagesDF = pagecountsDF2.cache().count()
println(s"RDD total: ${fmt.format(totalPagesRDD)}")
println(s"DF total:  ${fmt.format(totalPagesDF)}")

// COMMAND ----------

// MAGIC %md
// MAGIC ## Datasets
// MAGIC The easiest way to create a Dataset from scratch is from a DataFrame. Prior to Spark 2.0, we'd use the `SQLContext` for that:
// MAGIC
// MAGIC ```
// MAGIC val ds = sqlContext.read.text("dbfs:/tmp/pagecounts").as[String]
// MAGIC          |------------- makes a DataFrame ----------|
// MAGIC                                                       |--------|
// MAGIC                                                            |
// MAGIC                                                            +- converts to a Dataset
// MAGIC ```
// MAGIC
// MAGIC However, in 2.0, while that approach still works, we should prefer to use the `SparkSession`, available as `spark` in these notebooks and in the `spark-shell` Scala REPL.

// COMMAND ----------

// Notice that this returns a Dataset of Strings
val pagecountsDS = spark.read.text("dbfs:/tmp/pagecounts/").as[String]
pagecountsDS.take(3).foreach(println)

// COMMAND ----------

// MAGIC %md Of course, we could also just convert the existing `pagecountsDF` to a Dataset:

// COMMAND ----------

val pagecountsDS2 = pagecountsDF.as[(String, String, Long)]
pagecountsDS2.take(3).foreach(println)

// COMMAND ----------

// MAGIC %md Even better, though, let's make a Dataset that uses something more convenient than a tuple:

// COMMAND ----------

// Matching is done DataFrame column name -> case class field name
case class Edit(project: String, pageTitle: String, numberOfRequests: Long)
val pagecountsDS3 = pagecountsDF.as[Edit]

// COMMAND ----------

pagecountsDS3.take(4).foreach(println)

// COMMAND ----------

println(fmt.format(pagecountsDS3.count()))

// COMMAND ----------

// MAGIC %md ### I lied (a little)
// MAGIC
// MAGIC Prior to 2.0, `DataFrame` and `Dataset` were two different types. In 2.0, though, a `DataFrame` is just a type alias for `Dataset[Row]`. Thus, in 2.0,
// MAGIC when you start with a `DataFrame` and "convert" it to a `Dataset`, you're actually converting a `Dataset` of one type to a `Dataset` of another type.

// COMMAND ----------

pagecountsDF.getClass

// COMMAND ----------

// MAGIC %md You still have to get the types right on the conversions to Datasets, but once you have the Dataset, you have something that's type safe again.
// MAGIC
// MAGIC Once again, let's filter out the special pages, group by page title, and show to top 100 hits.

// COMMAND ----------

val pagecountsDS4 = pagecountsDS3.filter { e => (e.project == "en") && (! e.pageTitle.startsWith(".")) && (! isSpecialPage(e.pageTitle)) }.
                                  groupByKey { _.pageTitle }.  // GroupedDataset[String, DSEntry]
                                  reduceGroups { (e1, e2) => e1.copy(e1.project, e1.pageTitle, e1.numberOfRequests + e2.numberOfRequests) }.
                                  map(_._2). // skip the key; extract the value
                                  orderBy($"numberOfRequests".desc)

pagecountsDS4.take(100).foreach { e => println(s"${e.pageTitle}: ${e.numberOfRequests}") }

// COMMAND ----------

// MAGIC %md Let's cache this Dataset and, for good measure, compare the number of items with our RDD and DataFrame.

// COMMAND ----------

val totalPagesDS = pagecountsDS4.cache().count()
println(s"DF total:  ${fmt.format(totalPagesDF)}")
println(s"RDD total: ${fmt.format(totalPagesRDD)}")
println(s"DS total:  ${fmt.format(totalPagesDS)}")

// COMMAND ----------

pagecountsDS4.rdd.partitions.length

// COMMAND ----------

// MAGIC %md
// MAGIC ### The last little bit of caching
// MAGIC
// MAGIC So far, we have cached:
// MAGIC
// MAGIC * A filtered RDD of `(pageTitle, totalRequests)` tuples
// MAGIC * A DataFrame of `Row` objects
// MAGIC * A Dataset of `Edit` objects

// COMMAND ----------

// MAGIC %md Just for completeness, let's cache an RDD with `Edit` objects. Given the small amount of memory available to Community Edition clusters, we'll allow this cache to spill to disk if it has to.

// COMMAND ----------

import org.apache.spark.storage.StorageLevel
val pagecountsEditRDD = pagecountsRDD2.flatMap { line =>
  line.split("""\s+""") match {
    case Array(project, pageTitle, requests, _) => Some(Edit(project, pageTitle, requests.toLong))
    case _ => None
  }
}.
filter { e => (e.project == "en") && (! e.pageTitle.startsWith(".")) && (! isSpecialPage(e.pageTitle)) }
pagecountsEditRDD.setName("pagecountsEditRDD").persist(StorageLevel.MEMORY_AND_DISK).count()

// COMMAND ----------

// MAGIC %md The Spark UI's Storage tab now shows all of these in memory. The Dataset is compressed in memory by default, so it takes up much less space.

// COMMAND ----------

// MAGIC %md ## END OF DEMO

// COMMAND ----------


text:
layout: post
title: Chapter 2 - Onwards to the Human Kingdom
---

<iframe width= "560" height= "315" src= "https://www.youtube.com/embed/Z295qNy5NUU" frameborder= "0" allowfullscreen></iframe>


text:
import * as sinon from 'sinon';
import * as Qminder from '../../src/qminder-api';

describe('UserService', function () {
  const LOCATION_ID = 673;
  const USERS = [
    {
      id: 127,
      email: 'madli@example.com',
      firstName: 'Madli',
      lastName: 'Maasikas',
      picture: [
        {
          size: 'medium',
          url: 'http://www.google.com',
        },
      ],
    },
    {
      id: 245,
      email: 'kristjan@example.com',
      firstName: 'Kristjan',
      lastName: 'Kask',
      picture: [
        {
          size: 'medium',
          url: 'http://www.google.com',
        },
      ],
    },
  ];
  const pictureSizes = { small: 1, medium: 2, large: 3 };
  const pictureSort = (a: { size: string }, b: { size: string }) =>
    pictureSizes[a.size as 'small' | 'medium' | 'large'] -
    pictureSizes[b.size as 'small' | 'medium' | 'large'];
  let requestStub: sinon.SinonStub;

  beforeEach(function () {
    Qminder.setKey('EXAMPLE_API_KEY');
    Qminder.setServer('api.qminder.com');

    // Stub ApiBase.request to feed specific data to API
    requestStub = sinon.stub(Qminder.ApiBase, 'request');
  });

  afterEach(function () {
    (Qminder.ApiBase.request as sinon.SinonStub).restore();
  });

  describe('list()', function () {
    let usersReply: any;
    beforeEach(function (done) {
      requestStub
        .withArgs(`locations/${LOCATION_ID}/users`)
        .resolves({ data: USERS });

      Qminder.users.list(LOCATION_ID).then((users: Qminder.User[]) => {
        usersReply = users;
        done();
      });
    });

    it('returns the right user IDs', function () {
      const returnedIds = usersReply.map((user: Qminder.User) => user.id);
      const groundTruth = USERS.map((user) => user.id);

      for (let i = 0; i < groundTruth.length; i++) {
        expect(returnedIds[i]).toBe(groundTruth[i]);
      }
    });
    it('returns the right email addresses', function () {
      const returned = usersReply.map((user: Qminder.User) => user.email);
      const groundTruth = USERS.map((user) => user.email);

      for (let i = 0; i < groundTruth.length; i++) {
        expect(returned[i]).toBe(groundTruth[i]);
      }
    });
    it('returns the right first names', function () {
      const returned = usersReply.map((user: Qminder.User) => user.firstName);
      const groundTruth = USERS.map((user) => user.firstName);

      for (let i = 0; i < groundTruth.length; i++) {
        expect(returned[i]).toBe(groundTruth[i]);
      }
    });
    it('returns the right last names', function () {
      const returned = usersReply.map((user: Qminder.User) => user.lastName);
      const groundTruth = USERS.map((user) => user.lastName);

      for (let i = 0; i < groundTruth.length; i++) {
        expect(returned[i]).toBe(groundTruth[i]);
      }
    });
    it('returns the right pictures', function () {
      const returned = usersReply
        .map((user: Qminder.User) => user.picture[0])
        .sort(pictureSort);
      const groundTruth = USERS.map((user) => user.picture[0]).sort(
        pictureSort,
      );

      for (let i = 0; i < groundTruth.length; i++) {
        expect(returned[i].url).toBe(groundTruth[i].url);
      }
    });
  });
  describe('create()', function () {
    // Qminder.users.create(userdata)

    it('Sends the user roles as a JSON array', function () {
      const user: Omit<
        Qminder.User,
        'id' | 'desk' | 'selectedLocation' | 'picture'
      > = {
        email: 'test@qminder.com',
        firstName: 'Jon',
        lastName: 'Snow',
        roles: [
          {
            id: 1234,
            type: 'CLERK',
            location: 1234,
          },
          {
            id: 1235,
            type: 'CLERK',
            location: 1235,
          },
        ],
      };

      Qminder.users.create(user);
      expect(
        requestStub.calledWith(
          'users/',
          sinon.match({
            roles: JSON.stringify(user.roles),
          }),
        ),
      ).toBeTruthy();
    });
  });
  describe('details()', function () {
    // Qminder.users.details(userId|userEmail)
  });
  describe('addRole()', function () {
    // Qminder.users.addRole(user, role)
  });
  describe('setLines()', function () {
    it('Works with a list of Line IDs', function () {
      Qminder.users.setLines(123, [1, 2, 3, 4]);
      expect(
        requestStub.calledWith(
          'users/123/lines',
          sinon.match(JSON.stringify([1, 2, 3, 4])),
        ),
      ).toBeTruthy();
    });

    it('Works with a list of Lines', function () {
      const lines = [
        { id: 1, name: 'Test', color: '#fff', disabled: false },
        { id: 2, name: 'Test', color: '#fff', disabled: false },
        { id: 3, name: 'Test', color: '#f00', disabled: false },
      ];

      Qminder.users.setLines(123, lines);
      expect(
        requestStub.calledWith(
          'users/123/lines',
          sinon.match(JSON.stringify([1, 2, 3])),
        ),
      ).toBeTruthy();
    });

    it('Does not break when mixing IDs and objects', function () {
      const lines = [
        1,
        { id: 2, name: 'Test', color: '#fff', disabled: false },
        { id: 3, name: 'Test', color: '#f00', disabled: false },
      ];
      expect(() => Qminder.users.setLines(123, lines)).not.toThrow();
    });
  });
});


text:
package org.apache.qpid.proton.demo;

import java.util.ArrayList;
import java.util.List;

import org.apache.qpid.proton.engine.BaseHandler;
import org.apache.qpid.proton.engine.Collector;
import org.apache.qpid.proton.engine.Connection;
import org.apache.qpid.proton.engine.Delivery;
import org.apache.qpid.proton.engine.Event;
import org.apache.qpid.proton.engine.Link;
import org.apache.qpid.proton.engine.Sender;

public class Spout extends BaseHandler
{
    private int count;
    private int sent;
    private int settled;
    private boolean quiet;

    public Spout(int count, boolean quiet) {
        this.count = count;
        this.quiet = quiet;
    }

    @Override
    public void onLinkFlow(Event evt) {
        Link link = evt.getLink();
        if (link instanceof Sender) {
            Sender sender = (Sender) link;
            while ((sent < count) && sender.getCredit() > 0) {
                Delivery dlv = sender.delivery(String.format("spout-%s", sent).getBytes());

                Message msg = new Message(String.format("Hello World! [%s]", sent));
                byte[] bytes = msg.getBytes();
                sender.send(bytes, 0, bytes.length);
                sender.advance();

                if (!quiet) {
                    System.out.println(String.format("Sent %s to %s: %s", new String(dlv.getTag()),
                                                     sender.getTarget().getAddress(), msg));
                }
                sent++;
            }
        }
    }

    @Override
    public void onDelivery(Event evt) {
        Delivery dlv = evt.getDelivery();
        if (dlv.remotelySettled()) {
            if (!quiet) {
                System.out.println(String.format("Settled %s: %s", new String(dlv.getTag()), dlv.getRemoteState()));
            }
            dlv.settle();
            settled++;
        }

        if (settled >= count) {
            dlv.getLink().getSession().getConnection().close();
        }
    }

    @Override
    public void onConnectionRemoteClose(Event evt) {
        System.out.println("settled: " + settled);
    }

    public static void main(String[] argv) throws Exception {
        List<String> switches = new ArrayList<String>();
        List<String> args = new ArrayList<String>();
        for (String s : argv) {
            if (s.startsWith("-")) {
                switches.add(s);
            } else {
                args.add(s);
            }
        }

        boolean quiet = switches.contains("-q");
        String address = !args.isEmpty() && args.get(0).startsWith("/") ?
            args.remove(0) : "//localhost";
        int count = !args.isEmpty() ? Integer.parseInt(args.remove(0)) : 1;

        Collector collector = Collector.Factory.create();

        Spout spout = new Spout(count, quiet);

        Driver driver = new Driver(collector, spout);

        Pool pool = new Pool(collector);
        pool.outgoing(address, null);

        driver.run();
    }
}


text:
+++
date = "2016-04-20T12:00:00"
draft = false
image = "banners/getting-started.png"
tags = ["academic", "hugo"]
title = "Getting started with the Academic theme for Hugo"
math = true
summary = """
Create a beautifully simple academic or personal website in under 10 minutes.
"""
+++

The Academic theme enables you to easily create a beautifully simple academic or personal website using the [Hugo](https://gohugo.io) static site generator.

Key features:

- Designed for personal and academic staff/student use
- Customizable Biography, Publications, Projects, News/Blog, Teaching, and Contact widgets
- Write in [Markdown](https://github.com/adam-p/markdown-here/wiki/Markdown-Cheatsheet) for easy formatting and code highlighting, with [LaTeX](https://en.wikibooks.org/wiki/LaTeX/Mathematics) for mathematical expressions
- Academic linking (Scholar etc.), [Google Analytics](https://analytics.google.com), and [Disqus](https://disqus.com) comments
- Responsive and mobile friendly
- Simple and refreshing one page design

## Installation

1. [Install Hugo](https://georgecushen.com/create-your-website-with-hugo/#installing-hugo) and create a new website by typing the following commands in your *Terminal* or *Command Prompt* app:

        hugo new site my_website
        cd my_website

2. Install Academic theme with [git](https://help.github.com/articles/set-up-git/):

        git clone https://github.com/gcushen/hugo-academic.git themes/academic

    Or alternatively, [download Academic](https://github.com/gcushen/hugo-academic/archive/master.zip) and extract it into a `themes/academic` folder within your Hugo website.

3. If you are creating a new website, copy the contents of the `exampleSite` folder to your website root folder, overwriting existing files if necessary. The `exampleSite` folder contains an example config file and content to help you get started.

        cp -av themes/academic/exampleSite/* .

4. Start the Hugo server from your website root folder:

        hugo server --watch

    Now you can go to [localhost:1313](http://localhost:1313) and your new Academic themed website should appear.

5. Customize your website - refer to the *Getting Started* section below

6. Build your site by running the `hugo` command. Then [host it for free using Github Pages](https://georgecushen.com/create-your-website-with-hugo/). Or alternatively, copy the generated `public/` directory (by FTP, Rsync, etc.) to your production web server (such as your university's hosting service).

## Getting Started

Assuming you created a new website with the example content following the installation steps above, this section explores just a few more steps in order to customize it.

### Core parameters

The core parameters for the website can be edited in the `config.toml` configuration file:

- Set `baseurl` to your website URL (we recommend [GitHub Pages](https://georgecushen.com/create-your-website-with-hugo/) for free hosting)
- Set `title` to your desired website title such as your name
- The example Disqus commenting variable should be cleared (e.g. `disqusShortname = ""`) or set to your own [Disqus](https://disqus.com/) shortname to enable commenting
- Edit your details under `[params]`; these will be displayed mainly in the homepage *about* and *contact* widgets (if used). To disable a contact field, simply clear the value to `""`.
- Place a square cropped portrait photo named `portrait.jpg` into the `static/img/` folder, overwriting any defaults. Alternatively, you can edit the `avatar` filepath to point to a different image name or clear the value to disable the avatar feature.
- To enable LaTeX math for your site, set `math = true`
- Social/academic networking links are defined as multiples of `[[params.social]]`. They can be created or deleted as necessary.

### Introduce yourself

Edit your biography in the *about* widget `content/home/about.md` that you copied across from the `themes/academic/exampleSite/` folder. The research interests and qualifications are stored as `interests` and `education` variables. The academic qualifications are defined as multiples of `[[education.courses]]` and can be created or deleted as necessary. It's possible to completely hide the interests and education lists by deleting their respective variables.

### Customize homepage widgets

Each widget is responsible for a section on the homepage and contains further parameters that can be edited as desired. The parameters can be found in the preamble/frontmatter (between the pair of `+++`) for each widget located in the `content/home/` folder.

{{% alert note %}}
By default, publications will be displayed in a simple list. If you prefer a more detailed list with abstract and image, you can enable the detailed publication list on the homepage by setting `detailed_list = true` in `content/home/publications.md`.
{{% /alert %}}

### Add your content

Refer to our guide on [managing content]({{< ref "post/managing-content.md" >}}) to create your own homepage sections, publications, blog posts, and projects.

### Remove unused widgets and pages

[How to remove unused widgets and content pages]({{< ref "post/managing-content.md#removing-content" >}}).

### Customization & Upgrading

Continue reading below for advanced customization tips and instructions for keeping the theme up-to-date with any improvements that become available.

## Advanced customization

It is possible to carry out many customizations without touching any files in `themes/academic`, making it easier to upgrade the theme in the future.

### Navigation menu

The `[[menu.main]]` entries towards the bottom of `config.toml` define the navigation links at the top of the website. They can be added or removed as desired.

### Website icon

Save your main icon and mobile icon as square PNG images named `icon.png` and `apple-touch-icon.png`, respectively. Place them in your root `static/img/` folder.

### Theme color (CSS)

You can link custom CSS assets (relative to your root `static/css`) from your `config.toml` using `custom_css = ["custom.css"]`.

For example, lets make a green theme. First, define `custom_css = ["green.css"]` in `config.toml`. Then we can download the example [green theme](https://gist.github.com/gcushen/d5525a4506b9ccf83f2bce592a895495) and save it as `static/css/green.css`, relative to your website root (i.e. **not** in the `themes` directory).

### Analytics

To enable [Google Analytics](http://www.google.com/analytics), add your tracking code in `config.toml` similarly to `googleAnalytics = "UA-12345678-9"`.

### Third party and local scripts (JS)

To add a third party script, create a file named `head_custom.html` in a `layouts/partials/` folder at the root of your website (not in the `themes` folder). Any HTML code added to this file will be included within your website's `<head>`. Therefore, it's suitable for adding custom metadata or third party scripts specified with the *async* attribute.

Whereas for your own local scripts, you can link your local JS assets (relative to your root `static/js`) from your `config.toml` using `custom_js  = ["custom.js"]`.

### Permalinks

*Permalinks*, or *permanent links*, are URLs to individual pages and posts on your website. They are permanent web addresses which can be used to link to your content. Using Hugo's *permalinks* option these can be easily customized. For example, the blog post URL can be changed to the form *yourURL/2016/05/01/my-post-slug* by adding the following near the top of your `config.toml` (before `[params]` settings):

    [permalinks]
        post = "/:year/:month/:day/:slug"

Where `:slug` defaults to the filename of the post, excluding the file extension. However, slug may be overridden on a per post basis if desired, simply by setting `slug = "my-short-post-title"` in your post preamble.

## Upgrading

Feel free to *star* the project on [Github](https://github.com/gcushen/hugo-academic/) and monitor the [commits](https://github.com/gcushen/hugo-academic/commits/master) for updates.

Before upgrading the theme, it is recommended to make a backup of your entire website directory, or at least your `themes/academic` directory. You can also read about the [most recent milestones](https://github.com/gcushen/hugo-academic/releases) (but this doesn't necessarily reflect the latest *master* release).

Before upgrading for the first time, the remote *origin* repository should be renamed to *upstream*:

    $ cd themes/academic
    $ git remote rename origin upstream

To list available updates:

    $ cd themes/academic
    $ git fetch upstream
    $ git log --pretty=oneline --abbrev-commit --decorate HEAD..upstream/master

Then, upgrade by running:

    $ git pull upstream

If you have modified files in `themes/academic`, git will attempt to auto-merge changes. If conflicts are reported, you will need to manually edit the files with conflicts and add them back (`git add <filename>`).

If there are any issues after upgrading, you may wish to compare your site with the latest [example site](https://github.com/gcushen/hugo-academic/tree/master/exampleSite) to check if any settings changed.

## Feedback & Contributing

Please use the [issue tracker](https://github.com/gcushen/hugo-academic/issues) to let me know about any bugs or feature requests, or alternatively make a pull request.

For general questions about Hugo, there is a [Hugo discussion forum](http://discuss.gohugo.io).

## License

Copyright 2016 [George Cushen](https://georgecushen.com).

Released under the [MIT](https://github.com/gcushen/hugo-academic/blob/master/LICENSE.md) license.


text:
DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
cd $DIR/../../../../

PROJECT="teleport-test-170818"
TEMP="gs://teleport-test/temp/"
TEMPLATE="gs://teleport-test/templates/datastoreToBq"

if [[ -z $PROJECT ]]; then
  echo -n "Project Id hosting the templates (my-project-id): "
  read PROJECT
fi

if [[ -z $TEMP ]]; then
  echo -n "What is the temp location for the dataflow jobs (gs://...): "
  read TEMP
fi

if [[ -z $TEMPLATE ]]; then
  echo -n "Where to store this template (gs://...): "
  read TEMPLATE
fi

./gradlew clean build shadowJar -x test

java -jar build/libs/shadow-1.0-Alpha.jar \
  datastore_to_bq \
  --project=$PROJECT \
  --runner=DataflowRunner \
  --tempLocation=$TEMP \
  --templateLocation=$TEMPLATE

if [ $? -eq 0 ]; then
  echo "Success! Built Template"
else
  echo "Failed to build Template :'("
fi


text:
<md-toolbar layout="row" ng-if="$ctrl.authservice.logged">

        <md-button aria-label="menu" ng-click="$ctrl.open()" class="md-icon-button">
            <md-icon md-svg-icon="menu"></md-icon>
        </md-button>

        Admin Celebrities Store

                {{ $ctrl.authservice.user.displayName }}
                {{ $ctrl.authservice.user.email }}

</md-toolbar>


text:
<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN" "http://www.w3.org/TR/html4/loose.dtd">

<title>org.springframework.expression.spel.standard (Spring Framework 5.1.18.RELEASE API)</title>
<meta name="date" content="2020-09-15">
<link rel="stylesheet" type="text/css" href="../../../../../stylesheet.css" title="Style">

org.springframework.expression.spel.standard

Classes

SpelCompiler

SpelExpression

SpelExpressionParser


text:
package com.constantinnovationsinc.livemultimedia.utilities;

import android.annotation.TargetApi;
import android.os.Build;
import android.util.Size;

import java.util.Comparator;
/**
 * Compares two {@code Size}s based on their areas.
 */
@TargetApi(Build.VERSION_CODES.LOLLIPOP)
public class CompareSizesByArea implements Comparator<Size> {

    @Override
    public int compare(Size lhs, Size rhs) {
        // We cast here to ensure the multiplications won't overflow
        return Long.signum((long) lhs.getWidth() * lhs.getHeight() -
                (long) rhs.getWidth() * rhs.getHeight());
    }

}


text:
/**
 * Constant.cpp -
 * @author: Jonathan Beard
 * @version: Tue Jan  7 12:52:01 2014
 */
#include "Constant.hpp"

using namespace Node;

Constant::Constant() : StorageModifier( "Constant" )
{
   class_tree.addRelation( typeid( Node::StorageModifier ).hash_code(),
                           typeid( Node::Constant ).hash_code() );

}