text:
<?php
namespace Concrete\Core\Entity\File;

use Carbon\Carbon;
use Concrete\Core\Attribute\Key\FileKey;
use Concrete\Core\Attribute\ObjectTrait;
use Concrete\Core\Entity\Attribute\Value\FileValue;
use Concrete\Core\Entity\Attribute\Value\Value\Value;
use Concrete\Core\File\Exception\InvalidDimensionException;
use Concrete\Core\File\Image\Thumbnail\Path\Resolver;
use Concrete\Core\File\Image\Thumbnail\Thumbnail;
use Concrete\Core\File\Image\Thumbnail\Type\Type;
use Concrete\Core\File\Image\Thumbnail\Type\Version as ThumbnailTypeVersion;
use Concrete\Core\File\Menu;
use Concrete\Core\File\Type\TypeList as FileTypeList;
use Concrete\Core\Http\FlysystemFileResponse;
use Doctrine\Common\Collections\ArrayCollection;
use League\Flysystem\AdapterInterface;
use League\Flysystem\FileNotFoundException;
use Core;
use Database;
use Events;
use Imagine\Image\ImageInterface;
use Page;
use Permissions;
use stdClass;
use Symfony\Component\HttpFoundation\ResponseHeaderBag;
use User;
use View;
use Doctrine\ORM\Mapping as ORM;
use Concrete\Core\Support\Facade\Facade;

/**
 * @ORM\Entity
 * @ORM\Table(
 *     name="FileVersions",
 *     indexes={
 *     @ORM\Index(name="fvFilename", columns={"fvFilename"}),
 *     @ORM\Index(name="fvExtension", columns={"fvExtension"}),
 *     @ORM\Index(name="fvType", columns={"fvType"})
 *     }
 * )
 */
class Version
{
    use ObjectTrait {
        setAttribute as setFileVersionAttribute;
    }

    public function setAttribute($ak, $value)
    {
        $value = $this->setFileVersionAttribute($ak, $value);
        if (is_object($value)) {
            $this->attributes->add($value);
        }
    }

    const UT_REPLACE_FILE = 1;
    const UT_TITLE = 2;
    const UT_DESCRIPTION = 3;
    const UT_TAGS = 4;
    const UT_EXTENDED_ATTRIBUTE = 5;
    const UT_CONTENTS = 6;
    const UT_RENAME = 7;

    public function __construct()
    {
        $this->attributes = new ArrayCollection();
        $this->fvDateAdded = new \DateTime();
        $this->fvActivateDateTime = new \DateTime();
    }

    /**
     * /* @ORM\Id
     * @ORM\ManyToOne(targetEntity="File", inversedBy="versions")
     * @ORM\JoinColumn(name="fID", referencedColumnName="fID")
     *
     * @var \Concrete\Core\Entity\File\File
     */
    protected $file;
    /** @ORM\Id
     * @ORM\Column(type="integer")
     */
    protected $fvID = 0;
    /**
     * @ORM\Column(type="string")
     */
    protected $fvFilename = null;
    /**
     * @ORM\Column(type="string", nullable=true)
     */
    protected $fvPrefix;
    /**
     * @ORM\Column(type="datetime")
     */
    protected $fvDateAdded;
    /**
     * @ORM\Column(type="datetime")
     */
    protected $fvActivateDateTime;
    /**
     * @ORM\Column(type="boolean")
     */
    protected $fvIsApproved = false;
    /**
     * @ORM\Column(type="integer")
     */
    protected $fvAuthorUID = 0;
    /**
     * @ORM\Column(type="bigint")
     */
    protected $fvSize = 0;
    /**
     * @ORM\Column(type="integer")
     */
    protected $fvApproverUID = 0;
    /**
     * @ORM\Column(type="string", nullable=true)
     */
    protected $fvTitle = null;
    /**
     * @ORM\Column(type="text", nullable=true)
     */
    protected $fvDescription = null;
    /**
     * @ORM\Column(type="string", nullable=true)
     */
    protected $fvExtension = null;

    /**
     * @ORM\OneToMany(targetEntity="\Concrete\Core\Entity\Attribute\Value\FileValue",  mappedBy="version")
     * @ORM\JoinColumns({
     *   @ORM\JoinColumn(name="fID", referencedColumnName="fID"),
     *   @ORM\JoinColumn(name="fvID", referencedColumnName="fvID")
     * })
     */
    protected $attributes;

    /**
     * @ORM\Column(type="integer")
     */
    protected $fvType = 0;
    /**
     * @ORM\Column(type="text", nullable=true)
     */
    protected $fvTags = null;
    /**
     * @ORM\Column(type="boolean")
     */
    protected $fvHasListingThumbnail = false;
    /**
     * @ORM\Column(type="boolean")
     */
    protected $fvHasDetailThumbnail = false;

    public static function add(\Concrete\Core\Entity\File\File $file, $filename, $prefix, $data = array())
    {
        $u = new User();
        $uID = (isset($data['uID']) && $data['uID'] > 0) ? $data['uID'] : $u->getUserID();

        if ($uID < 1) {
            $uID = 0;
        }

        $fvTitle = (isset($data['fvTitle'])) ? $data['fvTitle'] : '';
        $fvDescription = (isset($data['fvDescription'])) ? $data['fvDescription'] : '';
        $fvTags = (isset($data['fvTags'])) ? self::cleanTags($data['fvTags']) : '';
        $fvIsApproved = (isset($data['fvIsApproved'])) ? $data['fvIsApproved'] : '1';

        $db = Database::get();
        $dh = Core::make('helper/date');
        $date = new Carbon($dh->getOverridableNow());

        $fv = new static();
        $fv->fvFilename = $filename;
        $fv->fvPrefix = $prefix;
        $fv->fvDateAdded = $date;
        $fv->fvIsApproved = (bool) $fvIsApproved;
        $fv->fvApproverUID = $uID;
        $fv->fvAuthorUID = $uID;
        $fv->fvActivateDateTime = $date;
        $fv->fvTitle = $fvTitle;
        $fv->fvDescription = $fvDescription;
        $fv->fvTags = $fvTags;
        $fv->file = $file;
        $fv->fvID = 1;

        $em = \ORM::entityManager();
        $em->persist($fv);
        $em->flush();

        $fve = new \Concrete\Core\File\Event\FileVersion($fv);
        Events::dispatch('on_file_version_add', $fve);

        return $fv;
    }

    public static function cleanTags($tagsStr)
    {
        $tagsArray = explode("\n", str_replace(array("\r", ","), "\n", $tagsStr));
        $cleanTags = array();
        foreach ($tagsArray as $tag) {
            if (!strlen(trim($tag))) {
                continue;
            }
            $cleanTags[] = trim($tag);
        }
        //the leading and trailing line break char is for searching: fvTag like %\ntag\n%
        return "\n" . implode("\n", $cleanTags) . "\n";
    }

    public function setFilename($filename)
    {
        $this->fvFilename = $filename;
    }

    public function getPrefix()
    {
        return $this->fvPrefix;
    }

    public function isApproved()
    {
        return $this->fvIsApproved;
    }

    public function getTagsList()
    {
        $tags = explode("\n", str_replace("\r", "\n", trim($this->getTags())));
        $clean_tags = array();
        foreach ($tags as $tag) {
            if (strlen(trim($tag))) {
                $clean_tags[] = trim($tag);
            }
        }

        return $clean_tags;
    }

    public function getTags()
    {
        return $this->fvTags;
    }

    /**
     * returns the File object associated with this FileVersion object.
     *
     * @return \File
     */
    public function getFile()
    {
        return $this->file;
    }

    public function setFile(\Concrete\Core\Entity\File\File $file)
    {
        $this->file = $file;
    }

    public function getFileID()
    {
        return $this->file->getFileID();
    }

    //returns an array of tags, instead of a string

    public function getFileVersionID()
    {
        return $this->fvID;
    }

    /**
     * Removes a version of a file. Note, does NOT remove the file because we don't know where the file might elsewhere be used/referenced.
     */
    public function delete($deleteFilesAndThumbnails = false)
    {
        $db = Database::get();
        $em = $db->getEntityManager();

        $values = array();

        $category = \Core::make('Concrete\Core\Attribute\Category\FileCategory');

        foreach ($this->attributes as $attribute) {
            $category->deleteValue($attribute);
        }

        $db->Execute("DELETE FROM FileVersionLog WHERE fID = ? AND fvID = ?", array($this->getFileID(), $this->fvID));

        $types = Type::getVersionList();

        if ($deleteFilesAndThumbnails) {
            try {
                foreach ($types as $type) {
                    $this->deleteThumbnail($type);
                }

                $fsl = $this->getFile()->getFileStorageLocationObject()->getFileSystemObject();
                $fre = $this->getFileResource();
                if ($fsl->has($fre->getPath())) {
                    $fsl->delete($fre->getPath());
                }
            } catch (FileNotFoundException $e) {
            }
        }

        $em = \ORM::entityManager();
        $em->remove($this);
        $em->flush();
    }

    /**
     * Deletes the thumbnail for the particular thumbnail type.
     */
    public function deleteThumbnail($type)
    {
        if (!($type instanceof ThumbnailTypeVersion)) {
            $type = ThumbnailTypeVersion::getByHandle($type);
        }
        $fsl = $this->getFile()->getFileStorageLocationObject()->getFileSystemObject();
        $path = $type->getFilePath($this);
        if ($fsl->has($path)) {
            $fsl->delete($path);
        }
    }

    /**
     * Returns an abstracted File object for the resource. NOT a concrete5 file object.
     *
     * @return \League\Flysystem\File
     */
    public function getFileResource()
    {
        $cf = Core::make('helper/concrete/file');
        $fs = $this->getFile()->getFileStorageLocationObject()->getFileSystemObject();
        $fo = $fs->get($cf->prefix($this->fvPrefix, $this->fvFilename));

        return $fo;
    }

    public function getMimeType()
    {
        $fre = $this->getFileResource();

        return $fre->getMimetype();
    }

    public function getSize()
    {
        return Core::make('helper/number')->formatSize($this->fvSize, 'KB');
    }

    public function getFullSize()
    {
        return $this->fvSize;
    }

    public function getAuthorName()
    {
        $ui = \UserInfo::getByID($this->fvAuthorUID);
        if (is_object($ui)) {
            return $ui->getUserDisplayName();
        }

        return t('(Unknown)');
    }

    public function getAuthorUserID()
    {
        return $this->fvAuthorUID;
    }

    /**
     * Gets the date a file version was added.
     *
     * @return string date formated like: 2009-01-01 00:00:00
     */
    public function getDateAdded()
    {
        return $this->fvDateAdded;
    }

    public function getExtension()
    {
        return $this->fvExtension;
    }

    public function setFileVersionID($fvID)
    {
        $this->fvID = $fvID;
    }

    /**
     * Takes the current value of the file version and makes a new one with the same values.
     */
    public function duplicate()
    {
        $db = Database::get();
        $em = \ORM::entityManager();
        $qq = $em->createQuery('SELECT max(v.fvID) FROM \Concrete\Core\Entity\File\Version v where v.file = :file');
        $qq->setParameter('file', $this->file);
        $fvID = $qq->getSingleScalarResult();
        ++$fvID;

        $fv = clone $this;
        $fv->fvID = $fvID;
        $fv->fvIsApproved = false;
        $fv->fvDateAdded = new \DateTime();

        $em->persist($fv);

        $this->deny();

        foreach ($this->attributes as $value) {
            $value = clone $value;
            /*
             * @var $value AttributeValue
             */
            $value->setVersion($fv);
            $em->persist($value);
        }

        $em->flush();

        $fe = new \Concrete\Core\File\Event\FileVersion($fv);
        Events::dispatch('on_file_version_duplicate', $fe);

        return $fv;
    }

    public function deny()
    {
        $this->fvIsApproved = false;
        $this->save();
        $fe = new \Concrete\Core\File\Event\FileVersion($this);
        Events::dispatch('on_file_version_deny', $fe);
    }

    protected function save($flush = true)
    {
        $em = \ORM::entityManager();
        $em->persist($this);
        if ($flush) {
            $em->flush();
        }
    }

    public function getType()
    {
        $ftl = $this->getTypeObject();
        if (is_object($ftl)) {
            return $ftl->getName();
        }
    }

    public function getTypeObject()
    {
        $fh = Core::make('helper/file');
        $ext = $fh->getExtension($this->fvFilename);

        $ftl = FileTypeList::getType($ext);

        return $ftl;
    }

    /**
     * Returns an array containing human-readable descriptions of everything that happened in this version.
     */
    public function getVersionLogComments()
    {
        $updates = array();
        $db = Database::get();
        $ga = $db->GetAll(
            'SELECT fvUpdateTypeID, fvUpdateTypeAttributeID FROM FileVersionLog WHERE fID = ? AND fvID = ? ORDER BY fvlID ASC',
            array($this->getFileID(), $this->getFileVersionID())
        );
        foreach ($ga as $a) {
            switch ($a['fvUpdateTypeID']) {
                case self::UT_REPLACE_FILE:
                    $updates[] = t('File');
                    break;
                case self::UT_TITLE:
                    $updates[] = t('Title');
                    break;
                case self::UT_DESCRIPTION:
                    $updates[] = t('Description');
                    break;
                case self::UT_TAGS:
                    $updates[] = t('Tags');
                    break;
                case self::UT_CONTENTS:
                    $updates[] = t('File Content');
                    break;
                case self::UT_RENAME:
                    $updates[] = t('File Name');
                    break;
                case self::UT_EXTENDED_ATTRIBUTE:
                    $val = $db->GetOne(
                        "SELECT akName FROM AttributeKeys WHERE akID = ?",
                        array($a['fvUpdateTypeAttributeID'])
                    );
                    if ($val != '') {
                        $updates[] = $val;
                    }
                    break;
            }
        }
        $updates = array_unique($updates);
        $updates1 = array();
        foreach ($updates as $val) {
            // normalize the keys
            $updates1[] = $val;
        }

        return $updates1;
    }

    public function updateTitle($title)
    {
        $this->fvTitle = $title;
        $this->save();
        $this->logVersionUpdate(self::UT_TITLE);
        $fe = new \Concrete\Core\File\Event\FileVersion($this);
        Events::dispatch('on_file_version_update_title', $fe);
    }

    public function logVersionUpdate($updateTypeID, $updateTypeAttributeID = 0)
    {
        $db = Database::get();
        $db->Execute(
            'INSERT INTO FileVersionLog (fID, fvID, fvUpdateTypeID, fvUpdateTypeAttributeID) VALUES (?, ?, ?, ?)',
            array(
                $this->getFileID(),
                $this->getFileVersionID(),
                $updateTypeID,
                $updateTypeAttributeID,
            )
        );
    }

    public function updateTags($tags)
    {
        $tags = self::cleanTags($tags);
        $this->fvTags = $tags;
        $this->save();
        $this->logVersionUpdate(self::UT_TAGS);
        $fe = new \Concrete\Core\File\Event\FileVersion($this);
        Events::dispatch('on_file_version_update_tags', $fe);
    }

    public function updateDescription($descr)
    {
        $this->fvDescription = $descr;
        $this->save();
        $this->logVersionUpdate(self::UT_DESCRIPTION);
        $fe = new \Concrete\Core\File\Event\FileVersion($this);
        Events::dispatch('on_file_version_update_description', $fe);
    }

    public function rename($filename)
    {
        $cf = Core::make('helper/concrete/file');
        $storage = $this->getFile()->getFileStorageLocationObject();
        $oldFilename = $this->fvFilename;
        if (is_object($storage)) {
            $path = $cf->prefix($this->fvPrefix, $oldFilename);
            $newPath = $cf->prefix($this->fvPrefix, $filename);
            $filesystem = $storage->getFileSystemObject();
            if ($filesystem->has($path)) {
                $filesystem->rename($path, $newPath);
            }
            $this->fvFilename = $filename;
            if ($this->fvTitle == $oldFilename) {
                $this->fvTitle = $filename;
            }
            $this->logVersionUpdate(self::UT_RENAME);
            $this->save();
        }
    }

    public function updateContents($contents)
    {
        $cf = Core::make('helper/concrete/file');
        $storage = $this->getFile()->getFileStorageLocationObject();
        if (is_object($storage)) {
            $path = $cf->prefix($this->fvPrefix, $this->fvFilename);
            $filesystem = $storage->getFileSystemObject();
            if ($filesystem->has($path)) {
                $filesystem->delete($path);
            }
            $filesystem->write($path, $contents);
            $this->logVersionUpdate(self::UT_CONTENTS);
            $fe = new \Concrete\Core\File\Event\FileVersion($this);
            Events::dispatch('on_file_version_update_contents', $fe);
            $this->refreshAttributes();
        }
    }

    public function updateFile($filename, $prefix)
    {
        $this->fvFilename = $filename;
        $this->fvPrefix = $prefix;
        $this->save();
        $this->logVersionUpdate(self::UT_REPLACE_FILE);
    }

    public function approve()
    {
        foreach ($this->file->getFileVersions() as $fv) {
            $fv->fvIsApproved = false;
            $fv->save(false);
        }

        $this->fvIsApproved = true;
        $this->save();

        $fe = new \Concrete\Core\File\Event\FileVersion($this);
        Events::dispatch('on_file_version_approve', $fe);

        $fo = $this->getFile();
        $fo->reindex();

        \Core::make('cache/request')->delete('file/version/approved/' . $this->getFileID());
    }

    /**
     * Return the contents of a file.
     */
    public function getFileContents()
    {
        $cf = Core::make('helper/concrete/file');
        $fsl = $this->getFile()->getFileStorageLocationObject();
        if (is_object($fsl)) {
            return $fsl->getFileSystemObject()->read($cf->prefix($this->fvPrefix, $this->fvFilename));
        }
    }

    /**
     * Returns a url that can be used to download a file, will force the download of all file types, even if your browser can display them.
     */
    public function getForceDownloadURL()
    {
        $c = Page::getCurrentPage();
        $cID = ($c instanceof Page) ? $c->getCollectionID() : 0;

        return View::url('/download_file', 'force', $this->getFileID(), $cID);
    }

    /**
     * Forces the download of a file.
     */
    public function forceDownload()
    {
        session_write_close();
        $fre = $this->getFileResource();

        $fs = $this->getFile()->getFileStorageLocationObject()->getFileSystemObject();
        $response = new FlysystemFileResponse($fre->getPath(), $fs);

        $response->setContentDisposition(ResponseHeaderBag::DISPOSITION_ATTACHMENT);
        $response->prepare(\Request::getInstance());

        ob_end_clean();
        $response->send();
        \Core::shutdown();
        exit;
    }

    public function getFileName()
    {
        return $this->fvFilename;
    }

    public function getRelativePath()
    {
        $cf = Core::make('helper/concrete/file');
        $fsl = $this->getFile()->getFileStorageLocationObject();
        if (is_object($fsl)) {
            $configuration = $fsl->getConfigurationObject();
            if ($configuration->hasRelativePath()) {
                return $configuration->getRelativePathToFile($cf->prefix($this->fvPrefix, $this->fvFilename));
            }
        }
    }

    public function getThumbnails()
    {
        $thumbnails = array();
        $types = Type::getVersionList();
        $width = $this->getAttribute('width');
        $file = $this->getFile();

        if (!$width || $width < 0) {
            throw new InvalidDimensionException($this->getFile(), $this, t('Invalid dimensions.'));
        }

        foreach ($types as $type) {
            if ($width <= $type->getWidth()) {
                continue;
            }

            $thumbnailPath = $type->getFilePath($this);
            $location = $file->getFileStorageLocationObject();
            $configuration = $location->getConfigurationObject();
            $filesystem = $location->getFileSystemObject();
            if ($filesystem->has($thumbnailPath)) {
                $thumbnails[] = new Thumbnail($type, $configuration->getPublicURLToFile($thumbnailPath));
            }
        }

        return $thumbnails;
    }

    public function getObjectAttributeCategory()
    {
        return \Core::make('\Concrete\Core\Attribute\Category\FileCategory');
    }

    /**
     * Necessary because getAttribute() returns the Value Value object, and this returns the
     * File Attribute Value object.
     *
     * @param $ak
     * @param $createIfNotExists bool
     *
     * @return mixed
     */
    public function getAttributeValueObject($ak, $createIfNotExists = false)
    {
        $handle = $ak;
        if (is_object($ak)) {
            $handle = $ak->getAttributeKeyHandle();
        }
        foreach ($this->attributes as $value) {
            if ($value->getAttributeKey()->getAttributeKeyHandle() == $handle) {
                return $value;
            }
        }

        if ($createIfNotExists) {
            if (!is_object($ak)) {
                $ak = FileKey::getByHandle($ak);
            }

            $attributeValue = new FileValue();
            $attributeValue->setVersion($this);
            $attributeValue->setAttributeKey($ak);

            return $attributeValue;
        }
    }

    public function rescanThumbnails()
    {
        if ($this->fvType != \Concrete\Core\File\Type\Type::T_IMAGE) {
            return false;
        }

        $app = Facade::getFacadeApplication();
        $width = $this->getAttribute('width');
        $types = Type::getVersionList();

        $fr = $this->getFileResource();
        try {
            $mimetype = $fr->getMimeType();
            $imageLibrary = \Image::getFacadeRoot();
            switch ($mimetype) {
                case 'image/svg+xml':
                case 'image/svg-xml':
                    if ($imageLibrary instanceof \Imagine\Gd\Imagine) {
                        try {
                            $imageLibrary = $app->make('image/imagick');
                        } catch (\Exception $x) {
                            return false;
                        }
                    }
                    break;
            }
            $image = $imageLibrary->load($fr->read());
            /* @var \Imagine\Imagick\Image $image */
            if (!$width) {
                $width = $image->getSize()->getWidth();
            }
            foreach ($types as $type) {

                // delete the file if it exists
                $this->deleteThumbnail($type);

                if ($width <= $type->getWidth()) {
                    continue;
                }

                $filesystem = $this->getFile()
                    ->getFileStorageLocationObject()
                    ->getFileSystemObject();

                $height = $type->getHeight();
                $thumbnailMode = ImageInterface::THUMBNAIL_OUTBOUND;
                if (!$height) {
                    $height = $type->getWidth();
                    $thumbnailMode = ImageInterface::THUMBNAIL_INSET;
                }
                $thumbnail = $image->thumbnail(new \Imagine\Image\Box($type->getWidth(), $height), $thumbnailMode);
                $thumbnailPath = $type->getFilePath($this);
                $thumbnailOptions = array();

                switch ($mimetype) {
                  case 'image/jpeg':
                    $thumbnailType = 'jpeg';
                    $thumbnailOptions = array('jpeg_quality' => \Config::get('concrete.misc.default_jpeg_image_compression'));
                    break;
                  case 'image/png':
                    $thumbnailType = 'png';
                    break;
                  case 'image/gif':
                    $thumbnailType = 'gif';
                    break;
                  case 'image/xbm':
                    $thumbnailType = 'xbm';
                    break;
                  case 'image/vnd.wap.wbmp':
                    $thumbnailType = 'wbmp';
                    break;
                  default:
                    $thumbnailType = 'png';
                    break;
                }

                $filesystem->write(
                    $thumbnailPath,
                    $thumbnail->get($thumbnailType, $thumbnailOptions),
                    array(
                        'visibility' => AdapterInterface::VISIBILITY_PUBLIC,
                        'mimetype' => $mimetype,
                    )
                );

                if ($type->getHandle() == \Config::get('concrete.icons.file_manager_listing.handle')) {
                    $this->fvHasListingThumbnail = true;
                }

                if ($type->getHandle() == \Config::get('concrete.icons.file_manager_detail.handle')) {
                    $this->fvHasDetailThumbnail = true;
                }

                unset($thumbnail);
                unset($filesystem);
            }
        } catch (\Imagine\Exception\InvalidArgumentException $e) {
            return false;
        }
        catch (\Imagine\Exception\RuntimeException $e) {
            return false;
        }
    }

    /**
     * @deprecated
     *
     * @param $level
     *
     * @return mixed
     */
    public function hasThumbnail($level)
    {
        switch ($level) {
            case 1:
                return $this->fvHasListingThumbnail;
            case 2:
                return $this->fvHasDetailThumbnail;
        }

        return false;
    }

    public function getDetailThumbnailImage()
    {
        if ($this->fvHasDetailThumbnail) {
            $type = Type::getByHandle(\Config::get('concrete.icons.file_manager_detail.handle'));
            $baseSrc = $this->getThumbnailURL($type->getBaseVersion());
            $doubledSrc = $this->getThumbnailURL($type->getDoubledVersion());

            return '<img src="' . $baseSrc . '" data-at2x="' . $doubledSrc . '" />';
        } else {
            return $this->getTypeObject()->getThumbnail();
        }
    }

    /**
     * Resolve a path using the default core path resolver.
     * Avoid using this method when you have access to your a resolver instance.
     *
     * @param $type
     * @return null|string
     */
    public function getThumbnailURL($type)
    {
        if (!($type instanceof ThumbnailTypeVersion)) {
            $type = ThumbnailTypeVersion::getByHandle($type);
        }

        /** @var Resolver $path_resolver */
        $path_resolver = Core::make('Concrete\Core\File\Image\Thumbnail\Path\Resolver');

        if ($path = $path_resolver->getPath($this, $type)) {
            return $path;
        }

        return $this->getURL();
    }

    /**
     * When given a thumbnail type versin object and a full path to a file on the server
     * the file is imported into the system as is as the thumbnail.
     *
     * @param ThumbnailTypeVersion $version
     * @param $path
     */
    public function importThumbnail(\Concrete\Core\File\Image\Thumbnail\Type\Version $version, $path)
    {
        $thumbnailPath = $version->getFilePath($this);
        $filesystem = $this->getFile()
            ->getFileStorageLocationObject()
            ->getFileSystemObject();
        if ($filesystem->has($thumbnailPath)) {
            $filesystem->delete($thumbnailPath);
        }

        $filesystem->write(
            $thumbnailPath,
            file_get_contents($path),
            array(
                'visibility' => AdapterInterface::VISIBILITY_PUBLIC,
                'mimetype' => 'image/jpeg',
            )
        );

        if ($version->getHandle() == \Config::get('concrete.icons.file_manager_listing.handle')) {
            $this->fvHasListingThumbnail = true;
        }

        if ($version->getHandle() == \Config::get('concrete.icons.file_manager_detail.handle')) {
            $this->fvHasDetailThumbnail = true;
        }

        $this->save();
    }

    /**
     * Returns a full URL to the file on disk.
     */
    public function getURL()
    {
        $cf = Core::make('helper/concrete/file');
        $fsl = $this->getFile()->getFileStorageLocationObject();
        if (is_object($fsl)) {
            $configuration = $fsl->getConfigurationObject();
            if ($configuration->hasPublicURL()) {
                return $configuration->getPublicURLToFile($cf->prefix($this->fvPrefix, $this->fvFilename));
            } else {
                return $this->getDownloadURL();
            }
        }
    }

    /**
     * Returns a URL that can be used to download the file. This passes through the download_file single page.
     */
    public function getDownloadURL()
    {
        $c = Page::getCurrentPage();
        $cID = ($c instanceof Page) ? $c->getCollectionID() : 0;

        return View::url('/download_file', $this->getFileID(), $cID);
    }

    /**
     * @return mixed
     */
    public function getAttributes()
    {
        return $this->attributes;
    }

    /**
     * Responsible for taking a particular version of a file and rescanning all its attributes
     * This will run any type-based import routines, and store those attributes, generate thumbnails,
     * etc...
     */
    public function refreshAttributes($rescanThumbnails = true)
    {
        $fh = Core::make('helper/file');
        $ext = $fh->getExtension($this->fvFilename);
        $ftl = FileTypeList::getType($ext);

        if (is_object($ftl)) {
            if ($ftl->getCustomImporter() != false) {
                $this->fvGenericType = $ftl->getGenericType();
                $cl = $ftl->getCustomInspector();
                $cl->inspect($this);
            }
        }

        \ORM::entityManager()->refresh($this);

        $fsr = $this->getFileResource();
        if (!$fsr->isFile()) {
            return Importer::E_FILE_INVALID;
        }

        $size = $fsr->getSize();

        $this->fvExtension = $ext;
        $this->fvType = $ftl->getGenericType();
        if ($this->fvTitle === null) {
            $this->fvTitle = $this->getFilename();
        }
        $this->fvSize = $size;

        if ($rescanThumbnails) {
            $this->rescanThumbnails();
        }

        $this->save();

        $f = $this->getFile();
        $f->reindex();
    }

    public function getTitle()
    {
        return $this->fvTitle;
    }

    /**
     * Return a representation of the current FileVersion object as something easily serializable.
     */
    public function getJSONObject()
    {
        $r = new stdClass();
        $fp = new Permissions($this->getFile());
        $r->canCopyFile = $fp->canCopyFile();
        $r->canEditFileProperties = $fp->canEditFileProperties();
        $r->canEditFilePermissions = $fp->canEditFilePermissions();
        $r->canDeleteFile = $fp->canDeleteFile();
        $r->canReplaceFile = $fp->canEditFileContents();
        $r->canEditFileContents = $fp->canEditFileContents();
        $r->canRead = $fp->canRead();
        $r->canViewFile = $this->canView();
        $r->canEditFile = $this->canEdit();
        $r->url = $this->getURL();
        $r->urlInline = (string) View::url('/download_file', 'view_inline', $this->getFileID());
        $r->urlDownload = (string) View::url('/download_file', 'view', $this->getFileID());
        $r->title = $this->getTitle();
        $r->genericTypeText = $this->getGenericTypeText();
        $r->description = $this->getDescription();
        $r->fileName = $this->getFilename();
        $r->resultsThumbnailImg = $this->getListingThumbnailImage();
        $r->fID = $this->getFileID();
        $r->treeNodeMenu = new Menu($this->getfile());

        return $r;
    }

    /**
     * Checks current viewers for this type and returns true if there is a viewer for this type, false if not.
     */
    public function canView()
    {
        $to = $this->getTypeObject();
        if (is_object($to) && $to->getView() != '') {
            return true;
        }

        return false;
    }

    public function canEdit()
    {
        $to = $this->getTypeObject();
        if (is_object($to) && $to->getEditor() != '') {
            return true;
        }

        return false;
    }

    public function getGenericTypeText()
    {
        $to = $this->getTypeObject();

        return $to->getGenericTypeText($to->getGenericType());
    }

    //takes a string of comma or new line delimited tags, and puts them in the appropriate format

    public function getDescription()
    {
        return $this->fvDescription;
    }

    public function getListingThumbnailImage()
    {
        if ($this->fvHasListingThumbnail) {
            $type = Type::getByHandle(\Config::get('concrete.icons.file_manager_listing.handle'));
            $baseSrc = $this->getThumbnailURL($type->getBaseVersion());
            $doubledSrc = $this->getThumbnailURL($type->getDoubledVersion());
            $width = $type->getWidth();
            $height = $type->getHeight();
            return sprintf('<img class="ccm-file-manager-list-thumbnail" src="%s" data-at2x="%s">', $baseSrc, $doubledSrc);
        } else {
            return $this->getTypeObject()->getThumbnail();
        }
    }
}


text:
'use strict';

define(['angular', 'require'], function(angular, require){

    var app = angular.module('portal.notifications.directives', []);

    app.directive('notifications', function(){
        return {
            restrict : 'E',
            templateUrl : require.toUrl('./partials/notifications.html')
        }
    });

    app.directive('notificationBell', function(){
        return {
            restrict : 'E',
            scope : {
              directiveMode : '@mode',
              headerCtrl : '=headerCtrl'
            },
            templateUrl : require.toUrl('./partials/notification-bell.html')
        }
    });

    app.directive('notificationsListItem', function() {
      return {
        restrict: 'E',
        templateUrl: require.toUrl('./partials/notifications-list-item.html')
      }
    });

    return app;

});

redpajama_set_name: RedPajamaGithub


text:
<?php
/**
 * Created by PhpStorm.
 * User: aneeshdash
 * Date: 27/11/14
 * Time: 10:41 AM
 */

class Book extends Eloquent
{
    use SoftDeletingTrait;
    protected $table='books';

    function category() {
        return $this->belongsTo('Category');
    }

    function publication() {
        return $this->belongsTo('Publication');
    }

    function user() {
        return $this->belongsTo('User','issue');
    }

    function lostbook(){
        return $this->hasOne('LostBook');
    }

    function transaction(){
        return $this->hasMany('Transaction');
    }

}


text:
class AddCodeToGameLevel < ActiveRecord::Migration
  def change
    add_column :game_levels, :code, :string, null: false, length: 30, after: :description
  end

  def up
    GameLevel.destroy_all
  end
end


text:
package com.yahoo.athenz.zts.cert.impl;

import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import com.yahoo.athenz.auth.util.Crypto;
import com.yahoo.athenz.common.server.cert.CertSigner;
import com.yahoo.athenz.common.server.cert.CertSignerFactory;
import com.yahoo.athenz.zts.ZTSConsts;

import java.io.File;
import java.security.PrivateKey;
import java.security.cert.X509Certificate;

import org.bouncycastle.asn1.x500.X500Name;
import org.bouncycastle.operator.OperatorCreationException;
import org.bouncycastle.pkcs.PKCS10CertificationRequest;

import java.io.IOException;
import javax.security.auth.x500.X500Principal;

public class SelfCertSignerFactory implements CertSignerFactory {

    private static final Logger LOGGER = LoggerFactory.getLogger(SelfCertSignerFactory.class);

    @Override
    public CertSigner create() {

        // extract the private key for this self cert signer

        final String pKeyFileName = System.getProperty(ZTSConsts.ZTS_PROP_SELF_SIGNER_PRIVATE_KEY_FNAME);
        final String pKeyPassword = System.getProperty(ZTSConsts.ZTS_PROP_SELF_SIGNER_PRIVATE_KEY_PASSWORD);
        final String csrDn = System.getProperty(ZTSConsts.ZTS_PROP_SELF_SIGNER_CERT_DN,
                "cn=Self Signed Athenz CA,o=Athenz,c=US");

        if (pKeyFileName == null) {
            LOGGER.error("No private key path available for Self Cert Signer Factory");
            return null;
        }

        File caKey = new File(pKeyFileName);
        PrivateKey caPrivateKey = Crypto.loadPrivateKey(caKey, pKeyPassword);

        // now generate a CSR for our own CA and self sign it

        String csr;
        try {
            csr = Crypto.generateX509CSR(caPrivateKey, csrDn, null);
        } catch (OperatorCreationException | IOException ex) {
            LOGGER.error("Unable to generate X509 CSR for dn: " + csrDn
                    + ", error: " + ex.getMessage());
            return null;
        }

        // generate our self signed certificate

        X500Principal subject = new X500Principal(csrDn);
        X500Name issuer = X500Name.getInstance(subject.getEncoded());
        PKCS10CertificationRequest certReq = Crypto.getPKCS10CertRequest(csr);
        X509Certificate caCertificate = Crypto.generateX509Certificate(certReq,
                caPrivateKey, issuer, 30 * 24 * 60, true);

        return new SelfCertSigner(caPrivateKey, caCertificate);
    }
}


text:
from django.db import models
from django.utils import timezone
from users.models import User
from django.core.exceptions import ValidationError
from django.db import IntegrityError
from django.core.exceptions import ObjectDoesNotExist
import re
from django.core.validators import URLValidator

class TeamManager(models.Manager):
    def create_team(self, name, **kwargs):
        team = self.model()
        self.validate_name(name)
        team.name = name

        if "contact_number" in kwargs:
            self.validate_contact_number(kwargs["contact_number"])
            team.contact_number = kwargs["contact_number"]

        if "team_url" in kwargs:
            # It checks validation, scheme, unicode
            validate_team_url = URLValidator()
            validate_team_url(kwargs["team_url"])
            team.team_url = kwargs["team_url"]

        team.save(using = self._db)

        return team

    def validate_name(self, name):
        if len(name) < 1:
            raise ValidationError(
                "team name length should be at most 64",
            )
        elif len(name) > 64:
            raise ValidationError(
                "team name length should be at most 64",
            )

        # test code, not yet
        if bool(re.match('^[a-zA-Z0-9_-]+$', name)) is False:
            raise ValidationError(
                     "team name cannot contain things but alphabet, '_', number"
                  )

    def get_team(self, id):
        try:
            team = Team.objects.get(id = id)
        except ObjectDoesNotExist:
            raise ObjectDoesNotExist("The id is not contained in team DB")
        else:
            return team

    def delete_team(self, id):
        team = Team.objects.get_team(id)
        team.delete()

        if team.id is None:
            return True
        else:
            raise OperationalError("delete team failed")

    def validate_contact_number(self, contact_number):
        if len(contact_number) > 20:
            raise ValidationError("Contact number cannot contain more than 20 digits")
        if bool(re.match('^[0-9-]+$', contact_number)):
            pass
        else:
            raise ValidationError(("team contact number should not be with character"))

    # get team from foreign key of MtoM
    def get_project_team(self, id):
        try:
            team = self.get(id = id)
        except ObjectDoesNotExist:
            raise ObjectDoesNotExist(
                str(id) + "is not contained in project team DB"
            )
        else:
            return team

    # get wait_team from foreign key of MtoM
    def get_project_wait_team(self, id):
        try:
            wait_team = self.get(id = id)
        except ObjectDoesNotExist:
            raise ObjectDoesNotExist(
                str(id) + "is not contained in project wait team DB"
            )
        else:
            return wait_team

    def update_team(self, id, **kwargs):
        team = Team.objects.get_team(id)

        if 'contact_number' in kwargs:
            self.validate_contact_number(kwargs['contact_number'])
            team.contact_number = kwargs['contact_number']

        if 'team_url' in kwargs:
            validate_team_url = URLValidator()
            validate_team_url(kwargs['team_url'])
            team.team_url = kwargs['team_url']

        team.save(using = self._db)

class Team(models.Model):
    name = models.CharField(
        help_text = "Team name",
        max_length = 64,
        unique=True,
    )

    contact_number = models.CharField(
                        help_text = "Team contact_number to inform something or contact",
                        max_length = 20,
                     )

    team_url = models.URLField(
                      help_text = "Team team_url",
                      default = "",
                  )

    objects = TeamManager()

    wait_members = models.ManyToManyField(
                       User,
                       verbose_name = "team wait_members",
                       through = 'WaitList',
                       through_fields = ('team', 'wait_member'),
                       related_name="wait_member",
                   )

    members = models.ManyToManyField(
                  User,
                  verbose_name = "team members for team",
                  through = 'Membership',
                  through_fields = ('team', 'member'),
                  related_name="member",
              )

class MembershipManager(models.Manager):
    def create_membership(self, team, user):
        if user.__class__.__name__ is not "User":
            raise ValidationError("user argument must be User object")
        if team.__class__.__name__ is not "Team":
            raise ValidationError("team argument must be Team object")

        # Does the member already exist?
        try:
            team.members.get_member(id = user.id)
        except ObjectDoesNotExist:
            membership = self.model(
                team = team,
                member = user,
            )
            membership.save(using = self._db)
            return membership
        else:
            raise AlreadyMemberError(member.email + " is already the team member")

    # User -> membership_member -> leave_team
    def leave_team(self, id):
        try:
            membership = self.get(id = id)
        except ValidationError:
            raise ValidationError("The team id is not a member's belonged team")
        membership.delete()

    # Team -> membership_team -> exclude_member
    def exclude_member(self, id):
        try:
            membership = self.get(id = id)
        except ValidationError:
            raise ValidationError("The member is not a team member")
        membership.delete()

class Membership(models.Model):
    team = models.ForeignKey(
               Team,
               verbose_name = "membership team",
               related_name="membership_team",
           )
    member = models.ForeignKey(
                 User,
                 verbose_name = "membership member",
                 related_name="membership_member",
             )
    objects = MembershipManager()

    date_joined = models.DateTimeField(
                      help_text = "Membership date_joined when the member joined the team",
                      auto_now_add=True,
                  )
    is_admin = models.BooleanField(
                   help_text = "Membership is_admin",
                   default=False
               )

class WaitListManager(models.Manager):
    def create_wait_list(self, team, wait_member):
        if wait_member.__class__.__name__ is not "User":
            raise ValidationError("wait_member argument must be User object")
        if team.__class__.__name__ is not "Team":
            raise ValidationError("team argument must be Team object")

        # Does the member already exist?
        try:
            team.members.get_member(id = wait_member.id)
        except ObjectDoesNotExist:
            pass
        else:
            raise AlreadyMemberError(wait_member.email + "is already the team member")

        # Does the wait_member already exist?
        try:
            team.wait_members.get_member(id = wait_member.id)
        except ObjectDoesNotExist:
            wait_list = self.model(
                team = team,
                wait_member = wait_member,
            )
            wait_list.save(using = self._db)
            return wait_list
        else:
            raise AlreadyWaitMemberError(wait_member.email + " is already the team member")

    # User -> wait_list_user -> cancel_to_request_to_team
    def cancel_to_request_to_team(self, team):
        try:
            wait_list = self.get(team = team)
        except ValidationError:
            raise ValidationError("The team is not a member's requested team")
        wait_list.delete()

    # Team -> wait_list_team -> reject_to_join_team
    def reject_to_join_team(self, wait_member):
        try:
            wait_list = self.get(wait_member = wait_member)
        except ValidationError:
            raise ValidationError("The wait_wait_member is not a team member")
        wait_list.delete()

class WaitList(models.Model):
    team = models.ForeignKey(
               Team,
               verbose_name = "waitList team",
               related_name="wait_list_team",
           )
    wait_member = models.ForeignKey(
                      User,
                      verbose_name = "waitList wait_member",
                      related_name="wait_list_wait_member",
                  )
    date_requested = models.DateTimeField(
                         help_text = "WaitList date_requested when user " +
                             "send a request to join a team",
                         auto_now_add=True,
                     )

    objects = WaitListManager()

class AlreadyMemberError(Exception):
    pass

class AlreadyWaitMemberError(Exception):
    pass


text:
using System.Collections.Generic;

namespace AlgoKit.Collections.Heaps
{
    /// <summary>
    /// Represents an implicit heap-ordered complete quaternary tree.
    /// </summary>
    public class QuaternaryHeap<TKey, TValue> : ArrayHeap<TKey, TValue>
    {
        /// <summary>
        /// Creates an empty quaternary heap.
        /// </summary>
        /// <param name="comparer">The comparer used to determine whether one object should be extracted from the heap earlier than the other one.</param>
        public QuaternaryHeap(IComparer<TKey> comparer) : base(comparer, 4)
        {
        }

        /// <summary>
        /// Creates a quaternary heap out of given list of elements in linear time.
        /// </summary>
        /// <param name="items">The list of items to build a heap from.</param>
        /// <param name="comparer">The comparer used to determine whether one object should be extracted from the heap earlier than the other one.</param>
        public QuaternaryHeap(IComparer<TKey> comparer, IReadOnlyCollection<KeyValuePair<TKey, TValue>> items) : base(comparer, 4, items)
        {
        }
    }
}


text:
<?php

namespace Upc\Cards\Bundle\CardsBundle\Repository;

use Doctrine\ORM\EntityRepository;

/**
 * GroupCategoryRepository
 *
 * This class was generated by the Doctrine ORM. Add your own custom
 * repository methods below.
 */
class GroupCategoryRepository extends EntityRepository {

    public function getCategoriesHome() {

        $status = 1;

        $qb = $this->createQueryBuilder('cg')
                ->select('cg, c')
                ->innerJoin('cg.categories', 'c')
                ->where('cg.status = :c_status')
                ->andWhere('c.status = :c_status')
                ->setParameter('c_status', $status);;

        return $qb->getQuery()->getResult();

    }

}


text:
<!DOCTYPE html>

<meta charset=utf-8>
<title>37 fast ml-2013-09-03-Our followers and who else they follow</title>

fast_ml fast_ml-2013 fast_ml-2013-37 knowledge-graph by maker-knowledge-mining

37 fast ml-2013-09-03-Our followers and who else they follow

meta infos for this blog

Source: html

Introduction: Recently we hit 400 followers mark on Twitter. To celebrate we decided to do some data mining on  you , specifically to discover who our followers are and who else they follow. For your viewing pleasure we packaged the results nicely with Bootstrap. Here's some data science in action.

  Our followers
This table show our 20 most popular followers as measeared by their follower count. The occasional question marks stand for non-ASCII characters. Each link opens a new window.

 Followers
 Screen name
 Name
 Description

 8685
  pankaj
 Pankaj Gupta
 I lead the Personalization and Recommender Systems group at Twitter. Founded two startups in the past.

 5070
  ogrisel
 Olivier Grisel
 Datageek, contributor to scikit-learn, works with Python / Java / Clojure / Pig, interested in Machine Learning, NLProc, {Big|Linked|Open} Data and braaains!

 4582
  thuske
 thuske
 &

 4442
  ram
 Ram Ravichandran
 So

Summary: the most important sentenses genereted by tfidf model

sentIndex sentText sentNum sentScore

1 To celebrate we decided to do some data mining on  you , specifically to discover who our followers are and who else they follow. [sent-2, score-0.578]

2 Our followers   This table show our 20 most popular followers as measeared by their follower count. [sent-5, score-0.673]

3 Followers   Screen name   Name   Description               8685    pankaj    Pankaj Gupta   I lead the Personalization and Recommender Systems group at Twitter. [sent-8, score-0.099]

4 5070    ogrisel    Olivier Grisel   Datageek, contributor to scikit-learn, works with Python / Java / Clojure / Pig, interested in Machine Learning, NLProc, {Big|Linked|Open} Data and braaains! [sent-10, score-0.247]

5 4582    thuske    thuske   &           4442    ram    Ram Ravichandran   Software Research Engineer at Twitter. [sent-11, score-0.28]

6 Data crunchist @opendns                 2651    jtoy    jtoy   I study intelligence, ponder consciousness, build machine learning systems, program in clojure/R/ruby/python, and start companies. [sent-15, score-0.281]

7 co/iU3TYN84O8                 2231    bscofield    Ben Scofield   RailsConf Program Chair; Director of Ruby Central; mostly a nice guy                 1863    johncandido    John Candido   Developing predictive algorithms for financial markets and sports analytics. [sent-20, score-0.18]

8 ESPN Insider contributor                 1849    arnicas    Lynn Cherny   Data Consultant (Data Analysis/Mining/Graphical Data); Infovis, Python, R, Stats, Javascript, Gender, Science Fiction, TV, ex-researcher with Stanford PhD. [sent-22, score-0.148]

9 #b2b #saas #marketing #stanford                 1492    medicalphysics    medicalphysics   Serious Medical Physics and Fun ! [sent-25, score-0.198]

10 1329    nikolaypavlov    Nikolay Pavlov   Dreamer, entrepreneur and simply curious man. [sent-27, score-0.099]

11 1196    GaelVaroquaux    Gael Varoquaux   Researcher and geek: science + Python + data = brain imaging + scikit-learn + machine learning + Mayavi + physics (& traveling… sailing… hiking…)                 1085    treycausey    Trey Causey   Data scientist. [sent-29, score-0.582]

12 1026    statisticsblog    Statistics Blog   I blog about probability & statistics with a focus on Monte Carlo simulations & epistemology: http://t. [sent-33, score-0.284]

13 Who else they follow     This table shows the most common friends of our followers, that is who they follow. [sent-41, score-0.198]

14 41%   5070    ogrisel    Olivier Grisel   Datageek, contributor to scikit-learn, works with Python / Java / Clojure / Pig, interested in Machine Learning, NLProc, {Big|Linked|Open} Data and braaains! [sent-50, score-0.247]

15 34%   374533    googleresearch    Google Research   At Google, research is performed company wide; we conduct and leverage research to build large-scale systems that are used in the real world. [sent-52, score-0.393]

16 27%   13938    peteskomoroch    Peter Skomoroch   My mission is to create intelligent systems that help people make better decisions. [sent-59, score-0.131]

17 27%   73453    CompSciFact    Computer Science    One fact per day from computer science M-F from @JohnDCook. [sent-62, score-0.629]

18 One tip per day M-F plus occasional other tweets. [sent-64, score-0.267]

19 26%   108989    coursera    Coursera   Learning without limits. [sent-66, score-0.123]

20 Sometimes I blog about research and playing with machine learning                  20%   27247    ProbFact    Probability Fact   One probability fact per day M-F from @JohnDCook. [sent-74, score-0.568]

similar blogs computed by tfidf model

tfidf for this blog:

wordName wordTfidf (topN-words)

[('science', 0.331), ('followers', 0.287), ('scientist', 0.205), ('mining', 0.205), ('http', 0.18), ('contributor', 0.148), ('systems', 0.131), ('research', 0.131), ('computer', 0.13), ('coursera', 0.123), ('ben', 0.108), ('blog', 0.108), ('python', 0.099), ('braaains', 0.099), ('clojure', 0.099), ('datageek', 0.099), ('director', 0.099), ('entrepreneur', 0.099), ('follow', 0.099), ('geek', 0.099), ('graham', 0.099), ('jtoy', 0.099), ('kdnuggets', 0.099), ('lorica', 0.099), ('medicalphysics', 0.099), ('nlproc', 0.099), ('occasional', 0.099), ('ogrisel', 0.099), ('olivier', 0.099), ('pankaj', 0.099), ('pig', 0.099), ('rstats', 0.099), ('stats', 0.099), ('table', 0.099), ('thuske', 0.099), ('stanford', 0.098), ('statistics', 0.098), ('making', 0.098), ('predictive', 0.098), ('data', 0.086), ('day', 0.084), ('per', 0.084), ('machine', 0.083), ('sports', 0.082), ('grisel', 0.082), ('engineer', 0.082), ('linked', 0.082), ('physics', 0.082), ('ram', 0.082), ('probability', 0.078)]

similar blogs list:

simIndex simValue blogId blogTitle

same-blog 1 1.0000002 37 fast ml-2013-09-03-Our followers and who else they follow

Introduction: Recently we hit 400 followers mark on Twitter. To celebrate we decided to do some data mining on  you , specifically to discover who our followers are and who else they follow. For your viewing pleasure we packaged the results nicely with Bootstrap. Here's some data science in action.

  Our followers
This table show our 20 most popular followers as measeared by their follower count. The occasional question marks stand for non-ASCII characters. Each link opens a new window.

 Followers
 Screen name
 Name
 Description

 8685
  pankaj
 Pankaj Gupta
 I lead the Personalization and Recommender Systems group at Twitter. Founded two startups in the past.

 5070
  ogrisel
 Olivier Grisel
 Datageek, contributor to scikit-learn, works with Python / Java / Clojure / Pig, interested in Machine Learning, NLProc, {Big|Linked|Open} Data and braaains!

 4582
  thuske
 thuske
 &

 4442
  ram
 Ram Ravichandran
 So

2 0.19533667 15 fast ml-2013-01-07-Machine learning courses online

Introduction: How do you learn machine learning? A good way to begin is to take an online course. These courses started appearing towards the end of 2011, first from Stanford University, now from  Coursera ,  Udacity ,  edX  and other institutions. There are very many of them, including a few about machine learning.   Here's a list:

 Introduction to Artificial Intelligence  by Sebastian Thrun and Peter Norvig. That was the first online class, and it contains two units on machine learning (units five and six). Both instructors work at Google. Sebastian Thrun is best known for building a self-driving car and Peter Norvig is a leading authority on AI, so they know what they are talking about. After the success of the class Sebastian Thrun quit Stanford to found Udacity, his online learning startup.

 Machine Learning  by Andrew Ng. Again, one of the first classes, by Stanford professor who started Coursera, the best known online learning provider today. Andrew Ng is a world class authority on m

3 0.13928397 62 fast ml-2014-05-26-Yann LeCun's answers from the Reddit AMA

Introduction: On May 15th Yann LeCun answered "ask me anything" questions on  Reddit . We hand-picked some of his thoughts and grouped them by topic for your enjoyment.
    Toronto, Montreal and New York
All three groups are strong and complementary.

Geoff (who spends more time at Google than in Toronto now) and Russ Salakhutdinov like RBMs and deep Boltzmann machines. I like the idea of Boltzmann machines (it's a beautifully simple concept) but it doesn't scale well. Also, I totally hate sampling.

Yoshua and his colleagues have focused a lot on various unsupervised learning, including denoising auto-encoders, contracting auto-encoders. They are not allergic to sampling like I am. On the application side, they have worked on text, not so much on images.

In our lab at NYU (Rob Fergus, David Sontag, me and our students and postdocs), we have been focusing on sparse auto-encoders for unsupervised learning. They have the advantage of scaling well. We have also worked on applications, mostly to v

4 0.13892294 41 fast ml-2013-10-09-Big data made easy

Introduction: An overview of key points about big data. This post was inspired by a very good article about big data by Chris Stucchio (linked below). The article is about hype and technology. We hate the hype.
    Big data is hype
Everybody talks about big data; nobody knows exactly what it is. That's pretty much the definition of hype.  Google Trends  suggest that the term took off at the beginning of 2011 (and the searches are coming mainly from Asia, curiously).

Now, to put things in context:

Big data is right there (or maybe not quite yet?) with other slogans like  web 2.0 ,  cloud computing  and  social media .
In effect,  big data  is a generic term for:

 data science
 machine learning
 data mining
 predictive analytics

and so on. Don't believe us? What about James Goodnight, the CEO of  SAS :

The term big data is being used today because computer analysts and journalists got tired of writing about cloud computing. Before cloud computing it was data warehousing or

5 0.10335237 22 fast ml-2013-03-07-Choosing a machine learning algorithm

Introduction: To celbrate the first 100 followers on Twitter, we asked them what would they like to read about here. One of the responders, Itamar Berger, suggested a topic: how to choose a ML algorithm for a task at hand. Well, what do we now?

Three things come to mind:

We'd try fast things first. In terms of speed, here's how we imagine the order:

 linear models
 trees, that is  bagged or boosted trees
 everything else*

We'd use something we are comfortable with. Learning new things is very exciting, however we'd ask ourselves a question: do we want to learn a new technique, or do we want a result?

We'd prefer something with fewer hyperparameters to set. More params means more tuning, that is training and re-training over and over, even if automatically. Random forests are hard to beat in this department. Linear models are pretty good too.

 A random forest scene, credit:  Jonathan MacGregor

*By "everything else" we mean "everything popular", mostly things like

6 0.056763109 28 fast ml-2013-05-12-And deliver us from Weka

7 0.050540872 47 fast ml-2013-12-15-A-B testing with bayesian bandits in Google Analytics

8 0.05006652 57 fast ml-2014-04-01-Exclusive Geoff Hinton interview

9 0.047261126 60 fast ml-2014-04-30-Converting categorical data into numbers with Pandas and Scikit-learn

10 0.045499906 27 fast ml-2013-05-01-Deep learning made easy

11 0.044519115 45 fast ml-2013-11-27-Object recognition in images with cuda-convnet

12 0.043692231 29 fast ml-2013-05-25-More on sparse filtering and the Black Box competition

13 0.041672055 46 fast ml-2013-12-07-13 NIPS papers that caught our eye

14 0.041509125 6 fast ml-2012-09-25-Best Buy mobile contest - full disclosure

15 0.041016258 3 fast ml-2012-09-01-Running Unix apps on Windows

16 0.04037514 40 fast ml-2013-10-06-Pylearn2 in practice

17 0.040317737 38 fast ml-2013-09-09-Predicting solar energy from weather forecasts plus a NetCDF4 tutorial

18 0.040180355 5 fast ml-2012-09-19-Best Buy mobile contest - big data

19 0.040002149 59 fast ml-2014-04-21-Predicting happiness from demographics and poll answers

20 0.038266771 10 fast ml-2012-11-17-The Facebook challenge HOWTO

similar blogs computed by lsi model

lsi for this blog:

topicId topicWeight

[(0, 0.174), (1, 0.1), (2, 0.233), (3, -0.098), (4, 0.122), (5, 0.18), (6, -0.053), (7, 0.187), (8, 0.417), (9, 0.179), (10, -0.215), (11, 0.094), (12, 0.027), (13, 0.057), (14, 0.124), (15, -0.019), (16, 0.043), (17, -0.09), (18, -0.056), (19, -0.009), (20, -0.06), (21, -0.045), (22, -0.154), (23, -0.021), (24, 0.157), (25, -0.003), (26, 0.192), (27, 0.083), (28, -0.099), (29, 0.077), (30, -0.046), (31, 0.112), (32, -0.121), (33, 0.008), (34, -0.062), (35, -0.018), (36, -0.039), (37, -0.014), (38, -0.201), (39, 0.029), (40, -0.162), (41, 0.108), (42, -0.177), (43, 0.181), (44, 0.117), (45, -0.071), (46, -0.053), (47, -0.157), (48, -0.102), (49, -0.014)]

similar blogs list:

simIndex simValue blogId blogTitle

same-blog 1 0.9742195 37 fast ml-2013-09-03-Our followers and who else they follow

Introduction: Recently we hit 400 followers mark on Twitter. To celebrate we decided to do some data mining on  you , specifically to discover who our followers are and who else they follow. For your viewing pleasure we packaged the results nicely with Bootstrap. Here's some data science in action.

  Our followers
This table show our 20 most popular followers as measeared by their follower count. The occasional question marks stand for non-ASCII characters. Each link opens a new window.

 Followers
 Screen name
 Name
 Description

 8685
  pankaj
 Pankaj Gupta
 I lead the Personalization and Recommender Systems group at Twitter. Founded two startups in the past.

 5070
  ogrisel
 Olivier Grisel
 Datageek, contributor to scikit-learn, works with Python / Java / Clojure / Pig, interested in Machine Learning, NLProc, {Big|Linked|Open} Data and braaains!

 4582
  thuske
 thuske
 &

 4442
  ram
 Ram Ravichandran
 So

2 0.43786439 15 fast ml-2013-01-07-Machine learning courses online

Introduction: How do you learn machine learning? A good way to begin is to take an online course. These courses started appearing towards the end of 2011, first from Stanford University, now from  Coursera ,  Udacity ,  edX  and other institutions. There are very many of them, including a few about machine learning.   Here's a list:

 Introduction to Artificial Intelligence  by Sebastian Thrun and Peter Norvig. That was the first online class, and it contains two units on machine learning (units five and six). Both instructors work at Google. Sebastian Thrun is best known for building a self-driving car and Peter Norvig is a leading authority on AI, so they know what they are talking about. After the success of the class Sebastian Thrun quit Stanford to found Udacity, his online learning startup.

 Machine Learning  by Andrew Ng. Again, one of the first classes, by Stanford professor who started Coursera, the best known online learning provider today. Andrew Ng is a world class authority on m

3 0.32187665 41 fast ml-2013-10-09-Big data made easy

Introduction: An overview of key points about big data. This post was inspired by a very good article about big data by Chris Stucchio (linked below). The article is about hype and technology. We hate the hype.
    Big data is hype
Everybody talks about big data; nobody knows exactly what it is. That's pretty much the definition of hype.  Google Trends  suggest that the term took off at the beginning of 2011 (and the searches are coming mainly from Asia, curiously).

Now, to put things in context:

Big data is right there (or maybe not quite yet?) with other slogans like  web 2.0 ,  cloud computing  and  social media .
In effect,  big data  is a generic term for:

 data science
 machine learning
 data mining
 predictive analytics

and so on. Don't believe us? What about James Goodnight, the CEO of  SAS :

The term big data is being used today because computer analysts and journalists got tired of writing about cloud computing. Before cloud computing it was data warehousing or

4 0.30539009 22 fast ml-2013-03-07-Choosing a machine learning algorithm

Introduction: To celbrate the first 100 followers on Twitter, we asked them what would they like to read about here. One of the responders, Itamar Berger, suggested a topic: how to choose a ML algorithm for a task at hand. Well, what do we now?

Three things come to mind:

We'd try fast things first. In terms of speed, here's how we imagine the order:

 linear models
 trees, that is  bagged or boosted trees
 everything else*

We'd use something we are comfortable with. Learning new things is very exciting, however we'd ask ourselves a question: do we want to learn a new technique, or do we want a result?

We'd prefer something with fewer hyperparameters to set. More params means more tuning, that is training and re-training over and over, even if automatically. Random forests are hard to beat in this department. Linear models are pretty good too.

 A random forest scene, credit:  Jonathan MacGregor

*By "everything else" we mean "everything popular", mostly things like

5 0.27515599 62 fast ml-2014-05-26-Yann LeCun's answers from the Reddit AMA

Introduction: On May 15th Yann LeCun answered "ask me anything" questions on  Reddit . We hand-picked some of his thoughts and grouped them by topic for your enjoyment.
    Toronto, Montreal and New York
All three groups are strong and complementary.

Geoff (who spends more time at Google than in Toronto now) and Russ Salakhutdinov like RBMs and deep Boltzmann machines. I like the idea of Boltzmann machines (it's a beautifully simple concept) but it doesn't scale well. Also, I totally hate sampling.

Yoshua and his colleagues have focused a lot on various unsupervised learning, including denoising auto-encoders, contracting auto-encoders. They are not allergic to sampling like I am. On the application side, they have worked on text, not so much on images.

In our lab at NYU (Rob Fergus, David Sontag, me and our students and postdocs), we have been focusing on sparse auto-encoders for unsupervised learning. They have the advantage of scaling well. We have also worked on applications, mostly to v

6 0.14912881 28 fast ml-2013-05-12-And deliver us from Weka

7 0.11983273 60 fast ml-2014-04-30-Converting categorical data into numbers with Pandas and Scikit-learn

8 0.11446805 6 fast ml-2012-09-25-Best Buy mobile contest - full disclosure

9 0.11380965 23 fast ml-2013-03-18-Large scale L1 feature selection with Vowpal Wabbit

10 0.11347861 38 fast ml-2013-09-09-Predicting solar energy from weather forecasts plus a NetCDF4 tutorial

11 0.11297888 25 fast ml-2013-04-10-Gender discrimination

12 0.10339084 59 fast ml-2014-04-21-Predicting happiness from demographics and poll answers

13 0.10198293 43 fast ml-2013-11-02-Maxing out the digits

14 0.1005353 24 fast ml-2013-03-25-Dimensionality reduction for sparse binary data - an overview

15 0.098162882 12 fast ml-2012-12-21-Tuning hyperparams automatically with Spearmint

16 0.096944436 29 fast ml-2013-05-25-More on sparse filtering and the Black Box competition

17 0.093103275 3 fast ml-2012-09-01-Running Unix apps on Windows

18 0.091894083 32 fast ml-2013-07-05-Processing large files, line by line

19 0.09031567 27 fast ml-2013-05-01-Deep learning made easy

20 0.081626445 21 fast ml-2013-02-27-Dimensionality reduction for sparse binary data

similar blogs computed by lda model

lda for this blog:

topicId topicWeight

[(15, 0.015), (30, 0.046), (31, 0.023), (69, 0.045), (71, 0.039), (78, 0.01), (79, 0.018), (84, 0.645), (92, 0.026), (99, 0.044)]

similar blogs list:

simIndex simValue blogId blogTitle

same-blog 1 0.96784127 37 fast ml-2013-09-03-Our followers and who else they follow

Introduction: Recently we hit 400 followers mark on Twitter. To celebrate we decided to do some data mining on  you , specifically to discover who our followers are and who else they follow. For your viewing pleasure we packaged the results nicely with Bootstrap. Here's some data science in action.

  Our followers
This table show our 20 most popular followers as measeared by their follower count. The occasional question marks stand for non-ASCII characters. Each link opens a new window.

 Followers
 Screen name
 Name
 Description

 8685
  pankaj
 Pankaj Gupta
 I lead the Personalization and Recommender Systems group at Twitter. Founded two startups in the past.

 5070
  ogrisel
 Olivier Grisel
 Datageek, contributor to scikit-learn, works with Python / Java / Clojure / Pig, interested in Machine Learning, NLProc, {Big|Linked|Open} Data and braaains!

 4582
  thuske
 thuske
 &

 4442
  ram
 Ram Ravichandran
 So

2 0.82811838 11 fast ml-2012-12-07-Predicting wine quality

Introduction: This post is as much about wine as it is about machine learning, so if you enjoy wine, like we do, you may find it especially interesting. Here's some  R and Matlab code , and if you want to get right to the point,  skip to the charts .

There's a book by Philipp Janert called  Data Analysis with Open Source Tools , which, by the way, we would recommend. From this book we found out about  the wine quality datasets . There are two, one for red wine and one for white wine, and they are interesting because they contain quality ratings (1 - 10) for a few thousands of wines, along with their physical and chemical properties. We could probably use these properties to predict a rating for a wine.   We'll be looking at white and red wine separately for the reasons you will see shortly.
  Principal component analysis for white wine
Janert performs a principal component analysis (PCA) and shows a resulting plot for white wine. What's interesting about this plot is that judging by the first tw

3 0.26331872 15 fast ml-2013-01-07-Machine learning courses online

Introduction: How do you learn machine learning? A good way to begin is to take an online course. These courses started appearing towards the end of 2011, first from Stanford University, now from  Coursera ,  Udacity ,  edX  and other institutions. There are very many of them, including a few about machine learning.   Here's a list:

 Introduction to Artificial Intelligence  by Sebastian Thrun and Peter Norvig. That was the first online class, and it contains two units on machine learning (units five and six). Both instructors work at Google. Sebastian Thrun is best known for building a self-driving car and Peter Norvig is a leading authority on AI, so they know what they are talking about. After the success of the class Sebastian Thrun quit Stanford to found Udacity, his online learning startup.

 Machine Learning  by Andrew Ng. Again, one of the first classes, by Stanford professor who started Coursera, the best known online learning provider today. Andrew Ng is a world class authority on m

4 0.18451078 41 fast ml-2013-10-09-Big data made easy

Introduction: An overview of key points about big data. This post was inspired by a very good article about big data by Chris Stucchio (linked below). The article is about hype and technology. We hate the hype.
    Big data is hype
Everybody talks about big data; nobody knows exactly what it is. That's pretty much the definition of hype.  Google Trends  suggest that the term took off at the beginning of 2011 (and the searches are coming mainly from Asia, curiously).

Now, to put things in context:

Big data is right there (or maybe not quite yet?) with other slogans like  web 2.0 ,  cloud computing  and  social media .
In effect,  big data  is a generic term for:

 data science
 machine learning
 data mining
 predictive analytics

and so on. Don't believe us? What about James Goodnight, the CEO of  SAS :

The term big data is being used today because computer analysts and journalists got tired of writing about cloud computing. Before cloud computing it was data warehousing or

5 0.17356807 9 fast ml-2012-10-25-So you want to work for Facebook

Introduction: Good news, everyone! There's a new contest on Kaggle -  Facebook is looking for talent . They won't pay, but just might interview.

This post is in a way a bonus for active readers because most visitors of fastml.com originally come from Kaggle forums. For this competition the forums are disabled to encourage  own work . To honor this, we won't publish any code. But  own work  doesn't mean  original work , and we wouldn't want to reinvent the wheel, would we?

The contest differs substantially from a Kaggle stereotype, if there is such a thing, in three major ways:

 there's no money prizes, as mentioned above
 it's not a real world problem, but rather an assignment to screen job candidates (this has important consequences, described below)
 it's not a typical machine learning project, but rather a broader AI exercise

You are given a graph of the internet, actually a snapshot of the graph for each of 15 time steps. You are also given a bunch of paths in this graph, which  a

6 0.16813244 62 fast ml-2014-05-26-Yann LeCun's answers from the Reddit AMA

7 0.14399756 53 fast ml-2014-02-20-Are stocks predictable?

8 0.13571264 46 fast ml-2013-12-07-13 NIPS papers that caught our eye

9 0.13033766 49 fast ml-2014-01-10-Classifying images with a pre-trained deep network

10 0.12954229 34 fast ml-2013-07-14-Running things on a GPU

11 0.12803948 45 fast ml-2013-11-27-Object recognition in images with cuda-convnet

12 0.12235162 48 fast ml-2013-12-28-Regularizing neural networks with dropout and with DropConnect

13 0.12198979 12 fast ml-2012-12-21-Tuning hyperparams automatically with Spearmint

14 0.12033345 3 fast ml-2012-09-01-Running Unix apps on Windows

15 0.11879335 55 fast ml-2014-03-20-Good representations, distance, metric learning and supervised dimensionality reduction

16 0.11756808 36 fast ml-2013-08-23-A bag of words and a nice little network

17 0.11724636 13 fast ml-2012-12-27-Spearmint with a random forest

18 0.11404555 16 fast ml-2013-01-12-Intro to random forests

19 0.111911 35 fast ml-2013-08-12-Accelerometer Biometric Competition

20 0.094170026 57 fast ml-2014-04-01-Exclusive Geoff Hinton interview


text:
package main

import (
	"fmt"
	"time"

	"github.com/shanebarnes/goto/logger"
)

type Metrics struct {
	timeStartNs   int64
	timeReportNs  int64 // Last report time
	byteReport    int64
	reportIntNs   int64 // Report interval in nanoseconds
	reportIntByte int64
	byteCount     int64
	tag           string
	buffer        []string
}

func MetricsNew(reportIntNs int64, reportIntByte int64, tag string) *Metrics {
	metrics := new(Metrics)
	metrics.timeStartNs = time.Now().UnixNano()
	metrics.timeReportNs = metrics.timeStartNs
	metrics.reportIntNs = reportIntNs
	metrics.reportIntByte = reportIntByte
	metrics.tag = tag
	metrics.byteReport = 0
	metrics.byteCount = 0
	metrics.Add(0)
	return metrics
}

func (m *Metrics) Add(bytes int64) {
	now := time.Now().UnixNano()
	m.byteCount = m.byteCount + bytes
	m.byteReport = m.byteReport + bytes

	if (m.reportIntNs > 0 && now >= m.timeReportNs) || (m.reportIntByte > 0 && m.byteReport >= m.reportIntByte) {
		avgBps := float64(m.byteCount) * 8. * 1000000000. / float64(now-m.timeStartNs)
		m.buffer = append(m.buffer, fmt.Sprintf("%.6f,%s,%d,%d,%d,%.6f", float64(now)/1000000000., m.tag, (now-m.timeStartNs)/1000, bytes, m.byteCount, avgBps/1000000.))

		m.byteReport = 0
		for m.timeReportNs <= now {
			m.timeReportNs = m.timeReportNs + m.reportIntNs
		}
	}
}

func (m *Metrics) Dump() {
	m.timeReportNs = time.Now().UnixNano()
	m.Add(0)

	for _, record := range m.buffer {
		logger.PrintlnInfo(record)
	}
}


text:
@protocol NSURLConnectionDelegate <NSObject>

@optional
- (id)connection:(id)arg1 willSendRequest:(id)arg2 redirectResponse:(id)arg3;
- (id)connection:(id)arg1 needNewBodyStream:(id)arg2;
- (BOOL)connection:(id)arg1 canAuthenticateAgainstProtectionSpace:(id)arg2;
- (void)connection:(id)arg1 didReceiveAuthenticationChallenge:(id)arg2;
- (void)connection:(id)arg1 willSendRequestForAuthenticationChallenge:(id)arg2;
- (void)connection:(id)arg1 didCancelAuthenticationChallenge:(id)arg2;
- (BOOL)connectionShouldUseCredentialStorage:(id)arg1;
- (void)connection:(id)arg1 didReceiveResponse:(id)arg2;
- (void)connection:(id)arg1 didReceiveData:(id)arg2;
- (void)connection:(id)arg1 didSendBodyData:(long long)arg2 totalBytesWritten:(long long)arg3 totalBytesExpectedToWrite:(long long)arg4;
- (void)connectionDidFinishLoading:(id)arg1;
- (void)connection:(id)arg1 didFailWithError:(id)arg2;
- (id)connection:(id)arg1 willCacheResponse:(id)arg2;
@end


text:
using System;
using System.Collections.Immutable;
using System.Linq;
using System.Threading.Tasks;
using Akka.Streams.Dsl;
using Akka.Streams.TestKit;
using Akka.Streams.TestKit.Tests;
using Akka.TestKit;
using FluentAssertions;
using Xunit;
using Xunit.Abstractions;
// ReSharper disable InvokeAsExtensionMethod

namespace Akka.Streams.Tests.Dsl
{

    public class GraphPartitionSpec : AkkaSpec
    {
        private ActorMaterializer Materializer { get; }

        public GraphPartitionSpec(ITestOutputHelper helper) : base(helper)
        {
            var settings = ActorMaterializerSettings.Create(Sys).WithInputBuffer(2,16);
            Materializer = ActorMaterializer.Create(Sys, settings);
        }

        [Fact]
        public void A_Partition_must_partition_to_three_subscribers()
        {
            this.AssertAllStagesStopped(() =>
            {
                var s = Sink.Seq<int>();
                var t = RunnableGraph.FromGraph(GraphDsl.Create(s, s, s, Tuple.Create, (b, sink1, sink2, sink3) =>
                {
                    var partition = b.Add(new Partition<int>(3, i => i > 3 ? 0 : (i < 3 ? 1 : 2)));
                    var source =
                        Source.From(Enumerable.Range(1, 5))
                            .MapMaterializedValue
                            <Tuple<Task<IImmutableList<int>>, Task<IImmutableList<int>>, Task<IImmutableList<int>>>>(
                                _ => null);

                    b.From(source).To(partition.In);
                    b.From(partition.Out(0)).To(sink1.Inlet);
                    b.From(partition.Out(1)).To(sink2.Inlet);
                    b.From(partition.Out(2)).To(sink3.Inlet);

                    return ClosedShape.Instance;
                })).Run(Materializer);

                var task = Task.WhenAll(t.Item1, t.Item2, t.Item3);
                task.Wait(TimeSpan.FromSeconds(3)).Should().BeTrue();
                task.Result[0].ShouldAllBeEquivalentTo(new[] {4, 5});
                task.Result[1].ShouldAllBeEquivalentTo(new[] {1, 2});
                task.Result[2].ShouldAllBeEquivalentTo(new[] {3});
            }, Materializer);
        }

        [Fact]
        public void A_Partition_must_complete_stage_after_upstream_completes()
        {
            this.AssertAllStagesStopped(() =>
            {
                var c1 = this.CreateSubscriberProbe<string>();
                var c2 = this.CreateSubscriberProbe<string>();

                RunnableGraph.FromGraph(GraphDsl.Create(b =>
                {
                    var partition = b.Add(new Partition<string>(2, s => s.Length > 4 ? 0 : 1));
                    var source = Source.From(new[] {"this", "is", "just", "another", "test"});

                    b.From(source).To(partition.In);
                    b.From(partition.Out(0)).To(Sink.FromSubscriber(c1));
                    b.From(partition.Out(1)).To(Sink.FromSubscriber(c2));

                    return ClosedShape.Instance;
                })).Run(Materializer);

                c1.Request(1);
                c2.Request(4);
                c1.ExpectNext("another");
                c2.ExpectNext("this", "is", "just", "test");
                c1.ExpectComplete();
                c2.ExpectComplete();
            }, Materializer);
        }

        [Fact]
        public void A_Partition_must_remember_first_pull_even_thought_first_element_target_another_out()
        {
            this.AssertAllStagesStopped(() =>
            {
                var c1 = this.CreateSubscriberProbe<int>();
                var c2 = this.CreateSubscriberProbe<int>();

                RunnableGraph.FromGraph(GraphDsl.Create(b =>
                {
                    var partition = b.Add(new Partition<int>(2, i => i < 6 ? 0 : 1));
                    var source = Source.From(new [] {6,3});

                    b.From(source).To(partition.In);
                    b.From(partition.Out(0)).To(Sink.FromSubscriber(c1));
                    b.From(partition.Out(1)).To(Sink.FromSubscriber(c2));

                    return ClosedShape.Instance;
                })).Run(Materializer);

                c1.Request(1);
                c1.ExpectNoMsg(TimeSpan.FromSeconds(1));
                c2.Request(1);
                c2.ExpectNext(6);
                c1.ExpectNext(3);
                c1.ExpectComplete();
                c2.ExpectComplete();
            }, Materializer);
        }

        [Fact]
        public void A_Partition_must_cancel_upstream_when_downstreams_cancel()
        {
            this.AssertAllStagesStopped(() =>
            {
                var p1 = this.CreatePublisherProbe<int>();
                var c1 = this.CreateSubscriberProbe<int>();
                var c2 = this.CreateSubscriberProbe<int>();

                RunnableGraph.FromGraph(GraphDsl.Create(b =>
                {
                    var partition = b.Add(new Partition<int>(2, i => i < 6 ? 0 : 1));
                    var source = Source.FromPublisher(p1.Publisher);

                    b.From(source).To(partition.In);
                    b.From(partition.Out(0))
                        .Via(Flow.Create<int>().Buffer(16, OverflowStrategy.Backpressure))
                        .To(Sink.FromSubscriber(c1));
                    b.From(partition.Out(1))
                        .Via(Flow.Create<int>().Buffer(16, OverflowStrategy.Backpressure))
                        .To(Sink.FromSubscriber(c2));

                    return ClosedShape.Instance;
                })).Run(Materializer);

                var p1Sub = p1.ExpectSubscription();
                var sub1 = c1.ExpectSubscription();
                var sub2 = c2.ExpectSubscription();
                sub1.Request(3);
                sub2.Request(3);
                p1Sub.SendNext(1);
                p1Sub.SendNext(8);
                c1.ExpectNext(1);
                c2.ExpectNext(8);
                p1Sub.SendNext(2);
                c1.ExpectNext(2);
                sub1.Cancel();
                sub2.Cancel();
                p1Sub.ExpectCancellation();
            }, Materializer);
        }

        [Fact]
        public void A_Partition_must_work_with_merge()
        {
            this.AssertAllStagesStopped(() =>
            {
                var s = Sink.Seq<int>();
                var input = new[] {5, 2, 9, 1, 1, 1, 10};

                var task = RunnableGraph.FromGraph(GraphDsl.Create(s, (b, sink) =>
                {
                    var partition = b.Add(new Partition<int>(2, i => i < 4 ? 0 : 1));
                    var merge = b.Add(new Merge<int>(2));
                    var source = Source.From(input).MapMaterializedValue<Task<IImmutableList<int>>>(_ => null);

                    b.From(source).To(partition.In);
                    b.From(partition.Out(0)).To(merge.In(0));
                    b.From(partition.Out(1)).To(merge.In(1));
                    b.From(merge.Out).To(sink.Inlet);

                    return ClosedShape.Instance;
                })).Run(Materializer);

                task.Wait(RemainingOrDefault).Should().BeTrue();
                task.Result.ShouldAllBeEquivalentTo(input);
            }, Materializer);
        }

        [Fact]
        public void A_Partition_must_stage_completion_is_waiting_for_pending_output()
        {
            this.AssertAllStagesStopped(() =>
            {
                var c1 = this.CreateSubscriberProbe<int>();
                var c2 = this.CreateSubscriberProbe<int>();

                RunnableGraph.FromGraph(GraphDsl.Create(b =>
                {
                    var partition = b.Add(new Partition<int>(2, i => i < 6 ? 0 : 1));
                    var source = Source.From(new[] { 6 });

                    b.From(source).To(partition.In);
                    b.From(partition.Out(0)).To(Sink.FromSubscriber(c1));
                    b.From(partition.Out(1)).To(Sink.FromSubscriber(c2));

                    return ClosedShape.Instance;
                })).Run(Materializer);

                c1.Request(1);
                c1.ExpectNoMsg(TimeSpan.FromSeconds(1));
                c2.Request(1);
                c2.ExpectNext(6);
                c1.ExpectComplete();
                c2.ExpectComplete();
            }, Materializer);
        }

        [Fact]
        public void A_Partition_must_fail_stage_if_partitioner_outcome_is_out_of_bound()
        {
            this.AssertAllStagesStopped(() =>
            {
                var c1 = this.CreateSubscriberProbe<int>();

                RunnableGraph.FromGraph(GraphDsl.Create(b =>
                {
                    var partition = b.Add(new Partition<int>(2, i => i < 0 ? -1 : 0));
                    var source = Source.From(new[] { -3 });

                    b.From(source).To(partition.In);
                    b.From(partition.Out(0)).To(Sink.FromSubscriber(c1));
                    b.From(partition.Out(1)).To(Sink.Ignore<int>().MapMaterializedValue(_ => NotUsed.Instance));

                    return ClosedShape.Instance;
                })).Run(Materializer);

                c1.Request(1);
                var error = c1.ExpectError();
                error.Should().BeOfType<PartitionOutOfBoundsException>();
                error.Message.Should()
                    .Be(
                        "partitioner must return an index in the range [0,1]. returned: [-1] for input [Int32].");
            }, Materializer);
        }
    }
}


text:
from __future__ import print_function

import unittest
import numpy as np
import paddle.fluid.core as core
from paddle.fluid.op import Operator
from op_test import OpTest

class TestSGDOp(OpTest):
    def setUp(self):
        self.op_type = "sgd"
        w = np.random.random((102, 105)).astype("float32")
        g = np.random.random((102, 105)).astype("float32")
        lr = np.array([0.1]).astype("float32")

        self.inputs = {'Param': w, 'Grad': g, 'LearningRate': lr}
        self.outputs = {'ParamOut': w - lr * g}

    def test_check_output(self):
        self.check_output()

class TestSparseSGDOp(unittest.TestCase):
    def check_with_place(self, place):
        scope = core.Scope()

        # create and initialize Grad Variable
        height = 10
        rows = [0, 4, 7]
        row_numel = 12

        grad_selected_rows = scope.var('Grad').get_selected_rows()
        grad_selected_rows.set_height(height)
        grad_selected_rows.set_rows(rows)
        np_array = np.ones((len(rows), row_numel)).astype("float32")
        np_array[0, 0] = 2.0
        np_array[2, 8] = 4.0

        grad_tensor = grad_selected_rows.get_tensor()
        grad_tensor.set(np_array, place)

        # create and initialize Param Variable
        param = scope.var('Param').get_tensor()
        param_array = np.full((height, row_numel), 5.0).astype("float32")
        param.set(param_array, place)

        # create and initialize LeraningRate Variable
        lr = scope.var('LearningRate').get_tensor()
        lr_array = np.full((1), 2.0).astype("float32")
        lr.set(lr_array, place)

        # create and run sgd operator
        sgd_op = Operator(
            "sgd",
            Param='Param',
            Grad='Grad',
            ParamOut='Param',
            LearningRate='LearningRate')
        sgd_op.run(scope, place)

        # get and compare result
        result_array = np.array(param)

        # rows[0] = 0, 5.0 - 2.0 * 2.0
        self.assertAlmostEqual(1.0, result_array[rows[0], 0])
        # rows[0] = 0, 5.0 - 2.0 * 1.0
        self.assertAlmostEqual(3.0, result_array[rows[0], 2])
        # 5.0 - 2.0 * 0.0
        self.assertAlmostEqual(5.0, result_array[1, 0])
        # rows[1] = 4, 5.0 - 2.0 * 1.0
        self.assertAlmostEqual(3.0, result_array[rows[1], 10])
        # 5.0 - 2.0 * 0.0
        self.assertAlmostEqual(5.0, result_array[5, 8])
        # rows[2] = 7, 5.0 - 2.0 * 1.0
        self.assertAlmostEqual(3.0, result_array[rows[2], 1])
        # rows[2] = 7, 5.0 - 2.0 * 4.0
        self.assertAlmostEqual(-3.0, result_array[rows[2], 8])

    def test_sparse_sgd(self):
        places = [core.CPUPlace()]
        if core.is_compiled_with_cuda():
            places.append(core.CUDAPlace(0))
        for place in places:
            self.check_with_place(place)

class TestSGDOpOptimizeSelectedRows(unittest.TestCase):
    def check_with_place(self, place):
        scope = core.Scope()

        row_width = 12
        # create and initialize Grad Variable
        grad_height = 10
        grad_rows = [0, 4, 7]

        grad_selected_rows = scope.var('Grad').get_selected_rows()
        grad_selected_rows.set_height(grad_height)
        grad_selected_rows.set_rows(grad_rows)
        grad_array = np.ones((len(grad_rows), row_width)).astype("float32")
        grad_array[0, 0] = 2.0
        grad_array[2, 8] = 4.0

        grad_tensor = grad_selected_rows.get_tensor()
        grad_tensor.set(grad_array, place)

        # create and initialize Param Variable
        # create and initialize W Variable
        param_rows = [0, 1, 2, 3, 4, 5, 6, 7]

        # init Param
        w_selected_rows = scope.var('Param').get_selected_rows()
        w_selected_rows.set_height(len(param_rows))
        w_selected_rows.set_rows(param_rows)
        w_selected_rows.sync_index()
        w_array = np.ones((len(param_rows), row_width)).astype("float32")
        for i in range(len(param_rows)):
            w_array[i] *= i
        w_tensor = w_selected_rows.get_tensor()
        w_tensor.set(w_array, place)

        w_before_optimize = np.array(w_tensor)

        # create and initialize LeraningRate Variable
        lr_value = 0.1
        lr = scope.var('LearningRate').get_tensor()
        lr_array = np.full((1), lr_value).astype("float32")
        lr.set(lr_array, place)

        # optimize with Python
        w_after_optimize = np.copy(w_before_optimize)
        for index, id in enumerate(grad_rows):
            w_after_optimize[id] = w_before_optimize[
                id] - lr_value * grad_array[index]

        # create and run sgd operator
        sgd_op = Operator(
            "sgd",
            Param='Param',
            Grad='Grad',
            ParamOut='Param',
            LearningRate='LearningRate')
        sgd_op.run(scope, place)

        # get and compare result
        result_array = np.array(w_tensor)
        assert (result_array == w_after_optimize).all()

    def test_sparse_parameter_sgd(self):
        places = [core.CPUPlace()]
        # do not support GPU kernel currently
        for place in places:
            self.check_with_place(place)

if __name__ == "__main__":
    unittest.main()


text:
<?php
	defined('IN_ADMIN') or exit('No permission resources.');
	include $this->admin_tpl('header', 'admin');
?>

<form name="myform" action="?m=video&c=video&a=<?php echo $_GET['a']?>" method="post" id="myform" enctype="multipart/form-data" onsubmit="return checkform()"><input type="hidden" name="userupload" value="1">

		<?php echo L('choose_videoes');?>

		<?php echo L('title');?>
		<input type="text" name="title" size="40" value="" id="title">

		<?php echo L('video_description')?>
		<textarea id="description" name="description" rows="5" cols="50"></textarea>

		<?php echo L('tags');?>
		<input type="text" id="keywords" name="keywords"  size="30" value=""> <?php echo L('separated_by_spaces');?>

<input type="hidden" name="vid" id="vid" value="0">
<input name="dosubmit" type="submit" value="<?php echo L('submit')?>" class="button" id="dosubmit">
</form>