text:
#include "library/constants.h"
#include "kernel/abstract.h"
#include "kernel/instantiate.h"
#include "kernel/kernel_exception.h"
#include "library/reducible.h"
#include "library/util.h"
#include "library/tactic/elaborate.h"
#include "library/tactic/expr_to_tactic.h"

namespace lean {
expr mk_generalize_tactic_expr(expr const & e, name const & id) {
    return mk_app(mk_constant(get_tactic_generalize_tac_name()),
                  e, mk_constant(id));
}

optional<proof_state> generalize_core(environment const & env, io_state const & ios, elaborate_fn const & elab,
                                      expr const & e, name const & x, proof_state const & s, name const & tac_name,
                                      bool intro) {
    proof_state new_s = s;
    if (auto new_e = elaborate_with_respect_to(env, ios, elab, new_s, e)) {
        substitution subst  = new_s.get_subst();
        goals const & gs    = new_s.get_goals();
        goal const & g      = head(gs);
        auto tc     = mk_type_checker(env);
        auto e_t_cs = tc->infer(*new_e);
        if (e_t_cs.second) {
            throw_tactic_exception_if_enabled(s, sstream() << "invalid '" << tac_name << "' tactic, unification constraints "
                                              "have been generated when inferring type");
            return none_proof_state(); // constraints were generated
        }
        expr e_t    = e_t_cs.first;
        expr t      = subst.instantiate(g.get_type());
        name n;
        if (is_constant(e))
            n = const_name(e);
        else if (is_local(e))
            n = local_pp_name(e);
        else
            n = x;
        expr new_t, new_m, new_val;
        expr to_check;
        if (intro) {
            buffer<expr> hyps;
            g.get_hyps(hyps);
            expr new_h = mk_local(mk_fresh_name(), get_unused_name(x, hyps), e_t, binder_info());
            new_t      = instantiate(abstract(t, *new_e), new_h);
            to_check   = Pi(new_h, new_t);
            new_m      = mk_metavar(mk_fresh_name(), Pi(hyps, Pi(new_h, new_t)));
            new_m      = mk_app(new_m, hyps);
            new_val    = mk_app(new_m, *new_e);
            new_m      = mk_app(new_m, new_h);
        } else {
            new_t    = mk_pi(n, e_t, abstract(t, *new_e));
            to_check = new_t;
            new_m    = g.mk_meta(mk_fresh_name(), new_t);
            new_val  = mk_app(new_m, *new_e);
        }
        try {
            check_term(*tc, g.abstract(to_check));
        } catch (kernel_exception const & ex) {
            std::shared_ptr<kernel_exception> ex_ptr(static_cast<kernel_exception*>(ex.clone()));
            throw_tactic_exception_if_enabled(s, [=](formatter const & fmt) {
                    format r = format("invalid '") + format(tac_name) + format("' tactic, type error");
                    r       += line();
                    r       += ex_ptr->pp(fmt);
                    return r;
                });
            return none_proof_state();
        }
        assign(subst, g, new_val);
        goal new_g(new_m, new_t);
        return some(proof_state(new_s, goals(new_g, tail(gs)), subst));
    }
    return none_proof_state();
}

optional<proof_state> generalize(environment const & env, io_state const & ios, elaborate_fn const & elab,
                                 expr const & e, name const & x, proof_state const & s) {
    return generalize_core(env, ios, elab, e, x, s, "generalize", false);
}

tactic generalize_tactic(elaborate_fn const & elab, expr const & e, name const & x) {
    return tactic01([=](environment const & env, io_state const & ios, proof_state const & s) {
            return generalize(env, ios, elab, e, x, s);
        });
}

void initialize_generalize_tactic() {
    register_tac(get_tactic_generalize_tac_name(),
                 [](type_checker &, elaborate_fn const & fn, expr const & e, pos_info_provider const *) {
                     check_tactic_expr(app_arg(app_fn(e)), "invalid 'generalize' tactic, invalid argument");
                     name id = tactic_expr_to_id(app_arg(e), "invalid 'generalize' tactic, argument must be an identifier");
                     return generalize_tactic(fn, get_tactic_expr_expr(app_arg(app_fn(e))), id);
                 });

    register_tac(get_tactic_generalizes_name(),
                 [](type_checker &, elaborate_fn const & fn, expr const & e, pos_info_provider const *) {
                     buffer<expr> args;
                     get_tactic_expr_list_elements(app_arg(e), args, "invalid 'generalizes' tactic, list of expressions expected");
                     if (args.empty())
                         return id_tactic();
                     tactic r = generalize_tactic(fn, args.back(), "x");
                     args.pop_back();
                     while (!args.empty()) {
                         r = then(generalize_tactic(fn, args.back(), "x"), r);
                         args.pop_back();
                     }
                     return r;
                 });
}

void finalize_generalize_tactic() {
}
}


text:
package edu.nudt.das.sansiro.core;

import javax.servlet.http.HttpSession;

/**
 * Created by yony on 17-6-9.
 */
public interface IMultiUserAble {
	Object getUser(HttpSession session);
	Object getRole(HttpSession session);
	Object getAuth();
}

redpajama_set_name: RedPajamaGithub


text:
// Copyright 2017 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

#include "base/task/thread_pool/test_utils.h"

#include <utility>

#include "base/bind.h"
#include "base/memory/raw_ptr.h"
#include "base/synchronization/condition_variable.h"
#include "base/task/thread_pool/pooled_parallel_task_runner.h"
#include "base/task/thread_pool/pooled_sequenced_task_runner.h"
#include "base/test/bind.h"
#include "base/threading/scoped_blocking_call_internal.h"
#include "base/threading/thread_restrictions.h"
#include "testing/gtest/include/gtest/gtest.h"

namespace base {
namespace internal {
namespace test {

namespace {

// A task runner that posts each task as a MockJobTaskSource that runs a single
// task. This is used to run ThreadGroupTests which require a TaskRunner with
// kJob execution mode. Delayed tasks are not supported.
class MockJobTaskRunner : public TaskRunner {
 public:
  MockJobTaskRunner(const TaskTraits& traits,
                    PooledTaskRunnerDelegate* pooled_task_runner_delegate)
      : traits_(traits),
        pooled_task_runner_delegate_(pooled_task_runner_delegate) {}

  MockJobTaskRunner(const MockJobTaskRunner&) = delete;
  MockJobTaskRunner& operator=(const MockJobTaskRunner&) = delete;

  // TaskRunner:
  bool PostDelayedTask(const Location& from_here,
                       OnceClosure closure,
                       TimeDelta delay) override;

 private:
  ~MockJobTaskRunner() override = default;

  const TaskTraits traits_;
  const raw_ptr<PooledTaskRunnerDelegate> pooled_task_runner_delegate_;
};

bool MockJobTaskRunner::PostDelayedTask(const Location& from_here,
                                        OnceClosure closure,
                                        TimeDelta delay) {
  DCHECK_EQ(delay, TimeDelta());  // Jobs doesn't support delayed tasks.

  if (!PooledTaskRunnerDelegate::MatchesCurrentDelegate(
          pooled_task_runner_delegate_)) {
    return false;
  }

  auto job_task = base::MakeRefCounted<MockJobTask>(std::move(closure));
  scoped_refptr<JobTaskSource> task_source = job_task->GetJobTaskSource(
      from_here, traits_, pooled_task_runner_delegate_);
  return pooled_task_runner_delegate_->EnqueueJobTaskSource(
      std::move(task_source));
}

scoped_refptr<TaskRunner> CreateJobTaskRunner(
    const TaskTraits& traits,
    MockPooledTaskRunnerDelegate* mock_pooled_task_runner_delegate) {
  return MakeRefCounted<MockJobTaskRunner>(traits,
                                           mock_pooled_task_runner_delegate);
}

}  // namespace

MockWorkerThreadObserver::MockWorkerThreadObserver()
    : on_main_exit_cv_(lock_.CreateConditionVariable()) {}

MockWorkerThreadObserver::~MockWorkerThreadObserver() {
  WaitCallsOnMainExit();
}

void MockWorkerThreadObserver::AllowCallsOnMainExit(int num_calls) {
  CheckedAutoLock auto_lock(lock_);
  EXPECT_EQ(0, allowed_calls_on_main_exit_);
  allowed_calls_on_main_exit_ = num_calls;
}

void MockWorkerThreadObserver::WaitCallsOnMainExit() {
  CheckedAutoLock auto_lock(lock_);
  while (allowed_calls_on_main_exit_ != 0)
    on_main_exit_cv_->Wait();
}

void MockWorkerThreadObserver::OnWorkerThreadMainExit() {
  CheckedAutoLock auto_lock(lock_);
  EXPECT_GE(allowed_calls_on_main_exit_, 0);
  --allowed_calls_on_main_exit_;
  if (allowed_calls_on_main_exit_ == 0)
    on_main_exit_cv_->Signal();
}

scoped_refptr<Sequence> CreateSequenceWithTask(
    Task task,
    const TaskTraits& traits,
    scoped_refptr<TaskRunner> task_runner,
    TaskSourceExecutionMode execution_mode) {
  scoped_refptr<Sequence> sequence =
      MakeRefCounted<Sequence>(traits, task_runner.get(), execution_mode);
  sequence->BeginTransaction().PushTask(std::move(task));
  return sequence;
}

scoped_refptr<TaskRunner> CreatePooledTaskRunnerWithExecutionMode(
    TaskSourceExecutionMode execution_mode,
    MockPooledTaskRunnerDelegate* mock_pooled_task_runner_delegate,
    const TaskTraits& traits) {
  switch (execution_mode) {
    case TaskSourceExecutionMode::kParallel:
      return CreatePooledTaskRunner(traits, mock_pooled_task_runner_delegate);
    case TaskSourceExecutionMode::kSequenced:
      return CreatePooledSequencedTaskRunner(traits,
                                             mock_pooled_task_runner_delegate);
    case TaskSourceExecutionMode::kJob:
      return CreateJobTaskRunner(traits, mock_pooled_task_runner_delegate);
    default:
      // Fall through.
      break;
  }
  ADD_FAILURE() << "Unexpected ExecutionMode";
  return nullptr;
}

scoped_refptr<TaskRunner> CreatePooledTaskRunner(
    const TaskTraits& traits,
    MockPooledTaskRunnerDelegate* mock_pooled_task_runner_delegate) {
  return MakeRefCounted<PooledParallelTaskRunner>(
      traits, mock_pooled_task_runner_delegate);
}

scoped_refptr<SequencedTaskRunner> CreatePooledSequencedTaskRunner(
    const TaskTraits& traits,
    MockPooledTaskRunnerDelegate* mock_pooled_task_runner_delegate) {
  return MakeRefCounted<PooledSequencedTaskRunner>(
      traits, mock_pooled_task_runner_delegate);
}

MockPooledTaskRunnerDelegate::MockPooledTaskRunnerDelegate(
    TrackedRef<TaskTracker> task_tracker,
    DelayedTaskManager* delayed_task_manager)
    : task_tracker_(task_tracker),
      delayed_task_manager_(delayed_task_manager) {}

MockPooledTaskRunnerDelegate::~MockPooledTaskRunnerDelegate() = default;

bool MockPooledTaskRunnerDelegate::PostTaskWithSequence(
    Task task,
    scoped_refptr<Sequence> sequence) {
  // |thread_group_| must be initialized with SetThreadGroup() before
  // proceeding.
  DCHECK(thread_group_);
  DCHECK(task.task);
  DCHECK(sequence);

  if (!task_tracker_->WillPostTask(&task, sequence->shutdown_behavior()))
    return false;

  if (task.delayed_run_time.is_null()) {
    PostTaskWithSequenceNow(std::move(task), std::move(sequence));
  } else {
    // It's safe to take a ref on this pointer since the caller must have a ref
    // to the TaskRunner in order to post.
    scoped_refptr<TaskRunner> task_runner = sequence->task_runner();
    delayed_task_manager_->AddDelayedTask(
        std::move(task),
        BindOnce(
            [](scoped_refptr<Sequence> sequence,
               MockPooledTaskRunnerDelegate* self, Task task) {
              self->PostTaskWithSequenceNow(std::move(task),
                                            std::move(sequence));
            },
            std::move(sequence), Unretained(this)),
        std::move(task_runner));
  }

  return true;
}

void MockPooledTaskRunnerDelegate::PostTaskWithSequenceNow(
    Task task,
    scoped_refptr<Sequence> sequence) {
  auto transaction = sequence->BeginTransaction();
  const bool sequence_should_be_queued = transaction.WillPushTask();
  RegisteredTaskSource task_source;
  if (sequence_should_be_queued) {
    task_source = task_tracker_->RegisterTaskSource(std::move(sequence));
    // We shouldn't push |task| if we're not allowed to queue |task_source|.
    if (!task_source)
      return;
  }
  transaction.PushTask(std::move(task));
  if (task_source) {
    thread_group_->PushTaskSourceAndWakeUpWorkers(
        {std::move(task_source), std::move(transaction)});
  }
}

bool MockPooledTaskRunnerDelegate::ShouldYield(const TaskSource* task_source) {
  return thread_group_->ShouldYield(task_source->GetSortKey(false));
}

bool MockPooledTaskRunnerDelegate::EnqueueJobTaskSource(
    scoped_refptr<JobTaskSource> task_source) {
  // |thread_group_| must be initialized with SetThreadGroup() before
  // proceeding.
  DCHECK(thread_group_);
  DCHECK(task_source);

  auto registered_task_source =
      task_tracker_->RegisterTaskSource(std::move(task_source));
  if (!registered_task_source)
    return false;
  auto transaction = registered_task_source->BeginTransaction();
  thread_group_->PushTaskSourceAndWakeUpWorkers(
      {std::move(registered_task_source), std::move(transaction)});
  return true;
}

void MockPooledTaskRunnerDelegate::RemoveJobTaskSource(
    scoped_refptr<JobTaskSource> task_source) {
  thread_group_->RemoveTaskSource(*task_source);
}

void MockPooledTaskRunnerDelegate::UpdatePriority(
    scoped_refptr<TaskSource> task_source,
    TaskPriority priority) {
  auto transaction = task_source->BeginTransaction();
  transaction.UpdatePriority(priority);
  thread_group_->UpdateSortKey(std::move(transaction));
}

void MockPooledTaskRunnerDelegate::UpdateJobPriority(
    scoped_refptr<TaskSource> task_source,
    TaskPriority priority) {
  UpdatePriority(std::move(task_source), priority);
}

void MockPooledTaskRunnerDelegate::SetThreadGroup(ThreadGroup* thread_group) {
  thread_group_ = thread_group;
}

MockJobTask::~MockJobTask() = default;

MockJobTask::MockJobTask(
    base::RepeatingCallback<void(JobDelegate*)> worker_task,
    size_t num_tasks_to_run)
    : worker_task_(std::move(worker_task)),
      remaining_num_tasks_to_run_(num_tasks_to_run) {}

MockJobTask::MockJobTask(base::OnceClosure worker_task)
    : worker_task_(base::BindRepeating(
          [](base::OnceClosure&& worker_task, JobDelegate*) mutable {
            std::move(worker_task).Run();
          },
          base::Passed(std::move(worker_task)))),
      remaining_num_tasks_to_run_(1) {}

size_t MockJobTask::GetMaxConcurrency(size_t /* worker_count */) const {
  return remaining_num_tasks_to_run_.load();
}

void MockJobTask::Run(JobDelegate* delegate) {
  worker_task_.Run(delegate);
  size_t before = remaining_num_tasks_to_run_.fetch_sub(1);
  DCHECK_GT(before, 0U);
}

scoped_refptr<JobTaskSource> MockJobTask::GetJobTaskSource(
    const Location& from_here,
    const TaskTraits& traits,
    PooledTaskRunnerDelegate* delegate) {
  return MakeRefCounted<JobTaskSource>(
      from_here, traits, base::BindRepeating(&test::MockJobTask::Run, this),
      base::BindRepeating(&test::MockJobTask::GetMaxConcurrency, this),
      delegate);
}

RegisteredTaskSource QueueAndRunTaskSource(
    TaskTracker* task_tracker,
    scoped_refptr<TaskSource> task_source) {
  auto registered_task_source =
      task_tracker->RegisterTaskSource(std::move(task_source));
  EXPECT_TRUE(registered_task_source);
  EXPECT_NE(registered_task_source.WillRunTask(),
            TaskSource::RunStatus::kDisallowed);
  return task_tracker->RunAndPopNextTask(std::move(registered_task_source));
}

void ShutdownTaskTracker(TaskTracker* task_tracker) {
  task_tracker->StartShutdown();
  task_tracker->CompleteShutdown();
}

}  // namespace test
}  // namespace internal
}  // namespace base


text:
<!DOCTYPE html>

    <meta charset="utf-8">
    <meta http-equiv="X-UA-Compatible" content="IE=edge">
    <meta name="viewport" content="width=device-width, initial-scale=1">
    <meta name="description" content="">
    <meta name="author" content="">

    <title>Bieter Eye Center - Contact Us</title>

    <link href="css/bootstrap.min.css" rel="stylesheet">

    <link href="css/business-casual.css" rel="stylesheet">

    <link href="http://fonts.googleapis.com/css?family=Open+Sans:300italic,400italic,600italic,700italic,800italic,400,300,600,700,800" rel="stylesheet" type="text/css">
    <link href="http://fonts.googleapis.com/css?family=Josefin+Slab:100,300,400,600,700,100italic,300italic,400italic,600italic,700italic" rel="stylesheet" type="text/css">

    <link rel="stylesheet" href="font-awesome-4.3.0/css/font-awesome.min.css">
    <link href="css/social-sharing.css" rel="stylesheet">

Bieter Eye Center

8617 W. Pt. Douglas Rd. S. Suite #110 Cottage Grove, MN 55016

Phone: 651-769-1020 Fax: 651-769-1021

                <button type="button" class="navbar-toggle" data-toggle="collapse" data-target="#bs-example-navbar-collapse-1">
                    Toggle navigation

                </button>

                Bieter Eye Center

                        Home

                        Contact Us

                        About

                        Services and Products

                        Optos

                        Why Use Our Optical?

                        Insurance Info

                        Promotions

                        Schedule An Appointment

                        Patient Forms

General Patient Information Form

Patient Financial Policy Form

Contact Us

Phone:
                        651-769-1020

Fax:
                        651-769-1021

Address:
                      8617 W. Pt. Douglas Rd. S. Suite #110 Cottage Grove, MN 55016

Copyright © Your Website 2014


text:
package com.asakusafw.runtime.stage.input;

import java.io.IOException;
import java.util.Iterator;
import java.util.List;

import org.apache.hadoop.mapreduce.InputFormat;
import org.apache.hadoop.mapreduce.InputSplit;
import org.apache.hadoop.mapreduce.RecordReader;
import org.apache.hadoop.mapreduce.TaskAttemptContext;
import org.apache.hadoop.util.ReflectionUtils;

import com.asakusafw.runtime.stage.input.StageInputSplit.Source;

/**
 * An implementation of Hadoop {@link RecordReader} for handling multiple Map operations.
 * @since 0.1.0
 * @version 0.2.6
 */
@SuppressWarnings("rawtypes")
public class StageInputRecordReader extends RecordReader {

    private static final RecordReader<?, ?> VOID = new RecordReader<Object, Object>() {

        @Override
        public void initialize(InputSplit split, TaskAttemptContext ctxt) {
            return;
        }

        @Override
        public boolean nextKeyValue() {
            return false;
        }

        @Override
        public Object getCurrentKey() {
            throw new IllegalStateException();
        }

        @Override
        public Object getCurrentValue() {
            throw new IllegalStateException();
        }

        @Override
        public float getProgress() {
            return 0f;
        }

        @Override
        public void close() {
            return;
        }
    };

    private Iterator<Source> sources;

    private TaskAttemptContext context;

    private RecordReader<?, ?> current;

    private boolean eof;

    private float progressPerSource;

    private float baseProgress;

    @Override
    public void initialize(
            InputSplit split,
            TaskAttemptContext taskContext) throws IOException, InterruptedException {
        assert split instanceof StageInputSplit;
        List<Source> sourceList = ((StageInputSplit) split).getSources();
        this.sources = sourceList.iterator();
        this.context = taskContext;
        this.progressPerSource = sourceList.isEmpty() ? 1f : 1f / sourceList.size();
        this.baseProgress = 0f;
        prepare();
    }

    private void prepare() throws IOException, InterruptedException {
        if (current != null) {
            baseProgress += progressPerSource;
            current.close();
        }
        if (sources.hasNext()) {
            Source next = sources.next();
            InputFormat<?, ?> format = ReflectionUtils.newInstance(next.getFormatClass(), context.getConfiguration());
            current = format.createRecordReader(next.getSplit(), context);
            current.initialize(next.getSplit(), context);
        } else {
            eof = true;
            current = VOID;
        }
    }

    @Override
    public boolean nextKeyValue() throws IOException, InterruptedException {
        while (eof == false) {
            if (current.nextKeyValue()) {
                return true;
            }
            prepare();
        }
        return false;
    }

    @Override
    public Object getCurrentKey() throws IOException, InterruptedException {
        return current.getCurrentKey();
    }

    @Override
    public Object getCurrentValue() throws IOException, InterruptedException {
        return current.getCurrentValue();
    }

    @Override
    public float getProgress() throws IOException, InterruptedException {
        float progress = current.getProgress();
        return baseProgress + progress * progressPerSource;
    }

    @Override
    public void close() throws IOException {
        if (current != null) {
            current.close();
        }
    }
}


text:
package com.github.obourgain.elasticsearch.http.response.entity;

import java.util.Map;
import lombok.Builder;
import lombok.Getter;

@Builder
@Getter
public class FieldStatistics {

    private int docCount;
    private long sumDocFreq;
    private long sumTtf;

    public static FieldStatistics fromMap(Map<String, Object> map) {
        int docCount = ((Number) map.get("doc_count")).intValue();
        long sumDocFreq = ((Number) map.get("sum_doc_freq")).longValue();
        long sumTtf = ((Number) map.get("sum_ttf")).longValue();
        return new FieldStatistics(docCount, sumDocFreq, sumTtf);
    }
}


text:
/* TEMPLATE GENERATED TESTCASE FILE
Filename: CWE761_Free_Pointer_Not_at_Start_of_Buffer__char_console_10.c
Label Definition File: CWE761_Free_Pointer_Not_at_Start_of_Buffer.label.xml
Template File: source-sinks-10.tmpl.c
*/
/*
 * @description
 * CWE: 761 Free Pointer not at Start of Buffer
 * BadSource: console Read input from the console
 * Sinks:
 *    GoodSink: free() memory correctly at the start of the buffer
 *    BadSink : free() memory not at the start of the buffer
 * Flow Variant: 10 Control flow: if(globalTrue) and if(globalFalse)
 *
 * */

#include "std_testcase.h"

#include <wchar.h>

#define SEARCH_CHAR 'S'

#ifndef OMITBAD

void CWE761_Free_Pointer_Not_at_Start_of_Buffer__char_console_10_bad()
{
    char * data;
    data = (char *)malloc(100*sizeof(char));
    data[0] = '\0';
    {
        /* Read input from the console */
        size_t dataLen = strlen(data);
        /* if there is room in data, read into it from the console */
        if (100-dataLen > 1)
        {
            /* POTENTIAL FLAW: Read data from the console */
            if (fgets(data+dataLen, (int)(100-dataLen), stdin) != NULL)
            {
                /* The next few lines remove the carriage return from the string that is
                 * inserted by fgets() */
                dataLen = strlen(data);
                if (dataLen > 0 && data[dataLen-1] == '\n')
                {
                    data[dataLen-1] = '\0';
                }
            }
            else
            {
                printLine("fgets() failed");
                /* Restore NUL terminator if fgets fails */
                data[dataLen] = '\0';
            }
        }
    }
    if(globalTrue)
    {
        /* FLAW: We are incrementing the pointer in the loop - this will cause us to free the
         * memory block not at the start of the buffer */
        for (; *data != '\0'; data++)
        {
            if (*data == SEARCH_CHAR)
            {
                printLine("We have a match!");
                break;
            }
        }
        free(data);
    }
}

#endif /* OMITBAD */

#ifndef OMITGOOD

/* goodB2G1() - use badsource and goodsink by changing globalTrue to globalFalse */
static void goodB2G1()
{
    char * data;
    data = (char *)malloc(100*sizeof(char));
    data[0] = '\0';
    {
        /* Read input from the console */
        size_t dataLen = strlen(data);
        /* if there is room in data, read into it from the console */
        if (100-dataLen > 1)
        {
            /* POTENTIAL FLAW: Read data from the console */
            if (fgets(data+dataLen, (int)(100-dataLen), stdin) != NULL)
            {
                /* The next few lines remove the carriage return from the string that is
                 * inserted by fgets() */
                dataLen = strlen(data);
                if (dataLen > 0 && data[dataLen-1] == '\n')
                {
                    data[dataLen-1] = '\0';
                }
            }
            else
            {
                printLine("fgets() failed");
                /* Restore NUL terminator if fgets fails */
                data[dataLen] = '\0';
            }
        }
    }
    if(globalFalse)
    {
        /* INCIDENTAL: CWE 561 Dead Code, the code below will never run */
        printLine("Benign, fixed string");
    }
    else
    {
        {
            size_t i;
            /* FIX: Use a loop variable to traverse through the string pointed to by data */
            for (i=0; i < strlen(data); i++)
            {
                if (data[i] == SEARCH_CHAR)
                {
                    printLine("We have a match!");
                    break;
                }
            }
            free(data);
        }
    }
}

/* goodB2G2() - use badsource and goodsink by reversing statements in if  */
static void goodB2G2()
{
    char * data;
    data = (char *)malloc(100*sizeof(char));
    data[0] = '\0';
    {
        /* Read input from the console */
        size_t dataLen = strlen(data);
        /* if there is room in data, read into it from the console */
        if (100-dataLen > 1)
        {
            /* POTENTIAL FLAW: Read data from the console */
            if (fgets(data+dataLen, (int)(100-dataLen), stdin) != NULL)
            {
                /* The next few lines remove the carriage return from the string that is
                 * inserted by fgets() */
                dataLen = strlen(data);
                if (dataLen > 0 && data[dataLen-1] == '\n')
                {
                    data[dataLen-1] = '\0';
                }
            }
            else
            {
                printLine("fgets() failed");
                /* Restore NUL terminator if fgets fails */
                data[dataLen] = '\0';
            }
        }
    }
    if(globalTrue)
    {
        {
            size_t i;
            /* FIX: Use a loop variable to traverse through the string pointed to by data */
            for (i=0; i < strlen(data); i++)
            {
                if (data[i] == SEARCH_CHAR)
                {
                    printLine("We have a match!");
                    break;
                }
            }
            free(data);
        }
    }
}

void CWE761_Free_Pointer_Not_at_Start_of_Buffer__char_console_10_good()
{
    goodB2G1();
    goodB2G2();
}

#endif /* OMITGOOD */

/* Below is the main(). It is only used when building this testcase on
   its own for testing or for building a binary to use in testing binary
   analysis tools. It is not used when compiling all the testcases as one
   application, which is how source code analysis tools are tested. */

#ifdef INCLUDEMAIN

int main(int argc, char * argv[])
{
    /* seed randomness */
    srand( (unsigned)time(NULL) );
#ifndef OMITGOOD
    printLine("Calling good()...");
    CWE761_Free_Pointer_Not_at_Start_of_Buffer__char_console_10_good();
    printLine("Finished good()");
#endif /* OMITGOOD */
#ifndef OMITBAD
    printLine("Calling bad()...");
    CWE761_Free_Pointer_Not_at_Start_of_Buffer__char_console_10_bad();
    printLine("Finished bad()");
#endif /* OMITBAD */
    return 0;
}

#endif


text:
from scrapy.spiders import Rule
from scrapy.linkextractors import LinkExtractor
from scrapy_redis.spiders import RedisCrawlSpider

class CrawlerRedis(RedisCrawlSpider):
    """Spider that reads urls from redis queue (myspider:start_urls)."""
    name = 'crawler_redis'
    redis_key = 'crawler:start_urls'

    """Spider that reads urls from redis queue when idle."""

    rules = (
        # follow all links
        Rule(LinkExtractor(), callback='parse_page', follow=True),
    )

    def __init__(self, *args, **kwargs):
        # Dynamically define the allowed domains list.
        domain = kwargs.pop('domain', '')
        self.allowed_domains = filter(None, domain.split(','))
        super(CrawlerRedis, self).__init__(*args, **kwargs)

    def parse_page(self, response):
        data = {
            'name': response.css('title::text').extract_first(),
            'url': response.url,
        }
        import  pprint
        pprint.pprint(data)
        return data


text:
<?xml version="1.0" encoding="UTF-8"?>
<extension type="module" version="2.5.0" client="site" method="upgrade">

    <name>mod_jit_testimonial</name>
    <creationDate>April 2014</creationDate>
    <author>Sharojit</author>
    <authorEmail>me@sharojit.com</authorEmail>
    <authorUrl>http://www.sharojit.com</authorUrl>
    <copyright>Copyright (C) 2015 Sharojit All rights reserved.</copyright>
    <license>GNU/GPL V2 for PHP files. CSS / JS are Copyrighted Commercial</license>
    <version>1.0</version>

    <description><![CDATA[
Jit Testimonial [ver. 1.1]

Advanced Testimonial module for Joomla! 3.x

Jit testimonial is one the product of the next generation of JIT extensions designed exclusively for Joomla 3+.

With the Jit Testimonial module, you can implement the module testimonial on websites. With built-in support for module embedding, the possibilities for implementing this new release are unlimited! Learn more at the

Jit Testimonial project website WWW.SHAROJIT.COM.

	]]></description>

    <languages>
        <language tag="en-GB">language/en-GB/en-GB.mod_jit_testimonial.ini
        </language>
        <language tag="en-GB">language/en-GB/en-GB.mod_jit_testimonial.sys.ini
        </language>
    </languages>

    <files>
        <folder>assets</folder>
        <folder>elements</folder>
        <folder>language</folder>
        <folder>tmpl</folder>
        <filename module="mod_jit_testimonial">mod_jit_testimonial.php</filename>
        <filename>mod_jit_testimonial.xml</filename>
        <filename>index.html</filename>
    </files>
    <config>
        <fields name="params" addfieldpath="/modules/mod_jit_testimonial/elements">
            <fieldset name="GENERAL_SETTING" label="MOD_GEN_SET_LABEL">
                <field name="img" type="media" label="IMG_LABEL" default="" description="IMG_LABEL_DESC"/>
                <field name="icon" type="text" label="ICON_LABLE" default="" description="ICON_LABLE_DESC"/>
                <field name="iconcolor" type="text" label="ICONCOLOR_LABLE" default="" description="ICONCOLOR_LABLE_DESC"/>
                <field name="title" type="text" label="TITLE_LABEL" default="" description="TITLE_LABEL_DESC"/>
                <field name="link" type="text" label="TITLE_LINK" default="" description="TITLE_LINK_DESC"/>
                <field name="titlecolor" type="text" label="TITLE_COLOR" default="" description="TITLE_COLOR_DESC"/>

                <field type="asset"/>
            </fieldset>

            <fieldset name="TESTIMONIAL_SETTING" label="MOD_TESTIMONIAL_LABEL">
                <field
                    name="box"
                    type="list"
                    label="BOX"
                    default="8"
                    description="BOX_DESC">
                    <option value="0">1</option>
                    <option value="1">2</option>
                    <option value="2">3</option>
                    <option value="3">4</option>
                    <option value="4">5</option>
                    <option value="5">6</option>
                    <option value="6">7</option>
                    <option value="7">8</option>
                    <option value="8">9</option>
                    <option value="9">10</option>
                    <option value="10">11</option>
                    <option value="11">12</option>
                </field>

                <field type="asset"/>
            </fieldset>

            <fieldset name="CLIENT1" label="CLIENT1_LABEL">
                <field name="image1" type="media" label="IMGL_LABEL" default="" description="IMGL_LABEL_DESC"/>
                <field name="testimonial1" type="text" label="TESTIMONIAL_LABEL" default="" description="TESTIMONIAL_LABEL_DESC"/>
                <field name="name1" type="text" label="CLIENT_LABEL" default="" description="CLIENT_LABEL_DESC"/>
                <field name="designation1" type="text" label="DESIGNATION_LABEL" default="" description="DESIGNATION_LABEL_DESC"/>
                <field name="bordercolor1" type="text" label="BORDER_LABEL" default="" description="BORDER_LABEL_DESC"/>

                <field type="asset"/>
            </fieldset>

            <fieldset name="CLIENT2" label="CLIENT2_LABEL">
                <field name="image2" type="media" label="IMGL_LABEL" default="" description="IMGL_LABEL_DESC"/>
                <field name="testimonial2" type="text" label="TESTIMONIAL_LABEL" default="" description="TESTIMONIAL_LABEL_DESC"/>
                <field name="name2" type="text" label="CLIENT_LABEL" default="" description="CLIENT_LABEL_DESC"/>
                <field name="designation2" type="text" label="DESIGNATION_LABEL" default="" description="DESIGNATION_LABEL_DESC"/>
                <field name="bordercolor2" type="text" label="BORDER_LABEL" default="" description="BORDER_LABEL_DESC"/>

                <field type="asset"/>
            </fieldset>

            <fieldset name="CLIENT3" label="CLIENT3_LABEL">
                <field name="image3" type="media" label="IMGL_LABEL" default="" description="IMGL_LABEL_DESC"/>
                <field name="testimonial3" type="text" label="TESTIMONIAL_LABEL" default="" description="TESTIMONIAL_LABEL_DESC"/>
                <field name="name3" type="text" label="CLIENT_LABEL" default="" description="CLIENT_LABEL_DESC"/>
                <field name="designation3" type="text" label="DESIGNATION_LABEL" default="" description="DESIGNATION_LABEL_DESC"/>
                <field name="bordercolor3" type="text" label="BORDER_LABEL" default="" description="BORDER_LABEL_DESC"/>

                <field type="asset"/>
            </fieldset>

            <fieldset name="CLIENT4" label="CLIENT4_LABEL">
                <field name="image4" type="media" label="IMGL_LABEL" default="" description="IMGL_LABEL_DESC"/>
                <field name="testimonial4" type="text" label="TESTIMONIAL_LABEL" default="" description="TESTIMONIAL_LABEL_DESC"/>
                <field name="name4" type="text" label="CLIENT_LABEL" default="" description="CLIENT_LABEL_DESC"/>
                <field name="designation4" type="text" label="DESIGNATION_LABEL" default="" description="DESIGNATION_LABEL_DESC"/>
                <field name="bordercolor4" type="text" label="BORDER_LABEL" default="" description="BORDER_LABEL_DESC"/>

                <field type="asset"/>
            </fieldset>

            <fieldset name="CLIENT5" label="CLIENT5_LABEL">
                <field name="image5" type="media" label="IMGL_LABEL" default="" description="IMGL_LABEL_DESC"/>
                <field name="testimonial5" type="text" label="TESTIMONIAL_LABEL" default="" description="TESTIMONIAL_LABEL_DESC"/>
                <field name="name5" type="text" label="CLIENT_LABEL" default="" description="CLIENT_LABEL_DESC"/>
                <field name="designation5" type="text" label="DESIGNATION_LABEL" default="" description="DESIGNATION_LABEL_DESC"/>
                <field name="bordercolor5" type="text" label="BORDER_LABEL" default="" description="BORDER_LABEL_DESC"/>

                <field type="asset"/>
            </fieldset>

            <fieldset name="CLIENT6" label="CLIENT6_LABEL">
                <field name="image6" type="media" label="IMGL_LABEL" default="" description="IMGL_LABEL_DESC"/>
                <field name="testimonial6" type="text" label="TESTIMONIAL_LABEL" default="" description="TESTIMONIAL_LABEL_DESC"/>
                <field name="name6" type="text" label="CLIENT_LABEL" default="" description="CLIENT_LABEL_DESC"/>
                <field name="designation6" type="text" label="DESIGNATION_LABEL" default="" description="DESIGNATION_LABEL_DESC"/>
                <field name="bordercolor6" type="text" label="BORDER_LABEL" default="" description="BORDER_LABEL_DESC"/>

                <field type="asset"/>
            </fieldset>

            <fieldset name="CLIENT7" label="CLIENT7_LABEL">
                <field name="image7" type="media" label="IMGL_LABEL" default="" description="IMGL_LABEL_DESC"/>
                <field name="testimonial7" type="text" label="TESTIMONIAL_LABEL" default="" description="TESTIMONIAL_LABEL_DESC"/>
                <field name="name7" type="text" label="CLIENT_LABEL" default="" description="CLIENT_LABEL_DESC"/>
                <field name="designation7" type="text" label="DESIGNATION_LABEL" default="" description="DESIGNATION_LABEL_DESC"/>
                <field name="bordercolor7" type="text" label="BORDER_LABEL" default="" description="BORDER_LABEL_DESC"/>

                <field type="asset"/>
            </fieldset>

            <fieldset name="CLIENT8" label="CLIENT8_LABEL">
                <field name="image8" type="media" label="IMGL_LABEL" default="" description="IMGL_LABEL_DESC"/>
                <field name="testimonial8" type="text" label="TESTIMONIAL_LABEL" default="" description="TESTIMONIAL_LABEL_DESC"/>
                <field name="name8" type="text" label="CLIENT_LABEL" default="" description="CLIENT_LABEL_DESC"/>
                <field name="designation8" type="text" label="DESIGNATION_LABEL" default="" description="DESIGNATION_LABEL_DESC"/>
                <field name="bordercolor8" type="text" label="BORDER_LABEL" default="" description="BORDER_LABEL_DESC"/>

                <field type="asset"/>
            </fieldset>

            <fieldset name="CLIENT9" label="CLIENT9_LABEL">
                <field name="image9" type="media" label="IMGL_LABEL" default="" description="IMGL_LABEL_DESC"/>
                <field name="testimonial9" type="text" label="TESTIMONIAL_LABEL" default="" description="TESTIMONIAL_LABEL_DESC"/>
                <field name="name9" type="text" label="CLIENT_LABEL" default="" description="CLIENT_LABEL_DESC"/>
                <field name="designation9" type="text" label="DESIGNATION_LABEL" default="" description="DESIGNATION_LABEL_DESC"/>
                <field name="bordercolor9" type="text" label="BORDER_LABEL" default="" description="BORDER_LABEL_DESC"/>

                <field type="asset"/>
            </fieldset>

            <fieldset name="CLIENT10" label="CLIENT10_LABEL">
                <field name="image10" type="media" label="IMGL_LABEL" default="" description="IMGL_LABEL_DESC"/>
                <field name="testimonial10" type="text" label="TESTIMONIAL_LABEL" default="" description="TESTIMONIAL_LABEL_DESC"/>
                <field name="name10" type="text" label="CLIENT_LABEL" default="" description="CLIENT_LABEL_DESC"/>
                <field name="designation10" type="text" label="DESIGNATION_LABEL" default="" description="DESIGNATION_LABEL_DESC"/>
                <field name="bordercolor10" type="text" label="BORDER_LABEL" default="" description="BORDER_LABEL_DESC"/>

                <field type="asset"/>
            </fieldset>

            <fieldset name="CLIENT11" label="CLIENT11_LABEL">
                <field name="image11" type="media" label="IMGL_LABEL" default="" description="IMGL_LABEL_DESC"/>
                <field name="testimonial11" type="text" label="TESTIMONIAL_LABEL" default="" description="TESTIMONIAL_LABEL_DESC"/>
                <field name="name11" type="text" label="CLIENT_LABEL" default="" description="CLIENT_LABEL_DESC"/>
                <field name="designation11" type="text" label="DESIGNATION_LABEL" default="" description="DESIGNATION_LABEL_DESC"/>
                <field name="bordercolor11" type="text" label="BORDER_LABEL" default="" description="BORDER_LABEL_DESC"/>

                <field type="asset"/>
            </fieldset>

            <fieldset name="CLIENT12" label="CLIENT12_LABEL">
                <field name="image12" type="media" label="IMGL_LABEL" default="" description="IMGL_LABEL_DESC"/>
                <field name="testimonial12" type="text" label="TESTIMONIAL_LABEL" default="" description="TESTIMONIAL_LABEL_DESC"/>
                <field name="name12" type="text" label="CLIENT_LABEL" default="" description="CLIENT_LABEL_DESC"/>
                <field name="designation12" type="text" label="DESIGNATION_LABEL" default="" description="DESIGNATION_LABEL_DESC"/>
                <field name="bordercolor12" type="text" label="BORDER_LABEL" default="" description="BORDER_LABEL_DESC"/>

                <field type="asset"/>
            </fieldset>

            <fieldset name="advanced">
                <field name="moduleclass_sfx" type="text" default="" label="MODSFX" description="MODSFX_DESC" />
                <field name="cache" type="list" default="1" label="COM_MODULES_FIELD_CACHING_LABEL" description="COM_MODULES_FIELD_CACHING_DESC">
                    <option value="1">JGLOBAL_USE_GLOBAL</option>
                    <option value="0">COM_MODULES_FIELD_VALUE_NOCACHING</option>
                </field>
                <field name="cache_time" type="text" default="900" label="COM_MODULES_FIELD_CACHE_TIME_LABEL" description="COM_MODULES_FIELD_CACHE_TIME_DESC" />
                <field name="cachemode" type="hidden" default="itemid">
                    <option value="itemid"></option>
                </field>
            </fieldset>
        </fields>
    </config>
</extension>


text:
import { Observable, Subject, Subscription } from "rxjs";

import { Repository } from "./repository";
import { GitHubApi } from '../github-api';
import { Http } from '@angular/http';

export class RepositoryChecker {

    private SECONDS = 45;

    private _hasLastCommit: boolean;
    private _isAlreadyNotified: boolean;
    private _repository: Repository;
    private _repositoryCheckerSubscription: Subscription;

    constructor(repository: Repository, private _gitHubApi: GitHubApi) {
        this._hasLastCommit = true;
        this._repository = repository;
        this._isAlreadyNotified = false;
    }

    public get repository(): Repository {
        return this._repository;
    }

    public isRepositoryHasNewCommitObservable(): Observable<boolean> {
        return Observable.interval(this.SECONDS * 1000).startWith(1).flatMap(() => {
            return this._gitHubApi.getRepositoryCommits(this._repository.fullname).map((c) => {
                let lastCommit = c[0];

                let isRepositoryHasNewCommit: boolean;

                if (this._isAlreadyNotified) {
                    isRepositoryHasNewCommit = false;
                } else {
                    isRepositoryHasNewCommit = lastCommit.sha === this._repository.lastCommitSha ? false : true;

                    if (isRepositoryHasNewCommit) {
                        this._hasLastCommit = false;
                        this._isAlreadyNotified = true;
                    } else {
                        this._hasLastCommit = true;
                    }
                }

                return isRepositoryHasNewCommit;
            });
        });
    }

    public get hasLastCommit(): boolean {
        return this._hasLastCommit;
    }

    public set lastCommitSha(commitSha: string) {
        if (commitSha) {
            this.repository.lastCommitSha = commitSha;
            this._hasLastCommit = true;
            this._isAlreadyNotified = false;
        } else {
            throw new Error("commitSha has null or undefined value.");
        }
    }
}


text:
import synthesize

def do():
    synthesize.say('My name is Jauffree.')


text:
<?php

namespace JVZ\CLI;

class CLI{

    private $argv;

    private $route;
    private $commands;
    private $options;

    private $controllers;
    private $controllersTree;

    private $before;
    private $after;
    private $on_error;
    private $main;

    public function __construct(){
        $this->clean();
        $this->setDefaultError();
    }

    public function on($controller,$function){
        preg_match_all("/\{.*?\}/",$controller,$params);

        /* _function and _params are not allowed */
        $controller = str_replace( array('_function','_params'), "", $controller);
        if(count($params)){
            $controller = str_replace( $params[0], "_param", $controller);
        }

        $this->controllers[ trim($controller) ] = array(
          '_function'   => $function,
          '_params' => isset($params[0]) ? $params[0] : array()
        );
    }

    public function getAllowedCommands(){
        return array_keys( $this->controllersTree );
    }

    public function getOptions(){
        return $this->options;
    }

    public function error( $function ){
        $this->on_error = $function;
    }

    public function main( $function ){
        $this->main  = $function;
    }

    public function before( $function ){
        $this->before[] = $function;
    }
    public function after( $function ){
        $this->after[] = $function;
    }

    public function run(){

        try{
            // parse command line
            $this->parseCommandParams();

            // run before
            $this->runBefore();

            // run controller
            $this->runController();

            //run after
            $this->runAfter();

        } catch (CliException $ex) {

            if( isset( $this->on_error ) ){
                call_user_func_array($this->on_error, array(  $ex ));
            }else{
                throw $ex;
            }
        }

    }

    /*
     * PRIVATE METHODS
     */
    private function setDefaultError(){

        $this->error(function( CliException $ex){
                echo '__________ ERROR __________'."\n";
                var_dump( $ex->getAdditionalData() );
            });

    }

    private function clean(){
        $this->before = array();
        $this->after = array();
        $this->on_error = null;

        $this->controllers = array();
        $this->controllersTree = array();

        $this->route = array();
        $this->options = array();
        $this->commands = array();
        $this->argv = array();
    }
    private function runBefore(){
        if( count($this->before)){
            foreach($this->before as $beforeFunct){
                call_user_func($beforeFunct);
            }
        }
    }
    private function runAfter(){
        if( count($this->after)){
            foreach($this->after as $afterFunct){
                call_user_func($afterFunct);
            }
        }
    }

    private function runController(){

        $this->buildTree();

        // search for the controller
        $found = array();       // this will hold found controller
        $foundParams = array(); // this will hold found arguments

        $returnArr = $this->controllersTree;
        $i=0;

        if ( !count($this->commands) && isset( $this->main ) ){
            return call_user_func( $this->main );
        }

        foreach($this->commands as $command){
            $i++;

            if( isset($returnArr[$command]) ){
                // found the match
                $found = $returnArr[$command];
                $returnArr = &$returnArr[$command];
            }else{

                // looking for arguments
                if( isset($returnArr['_param']) ){
                    // found the argument
                    $found = $returnArr['_param'];
                    $foundParams[] = $command;
                    $found['_params'] = $foundParams;
                    $returnArr = &$returnArr['_param'];
                }else{
                    // nothing found
                    throw new CliException("Invalid command",  CliException::INVALID_COMMAND, array('invalid_command'=>$command, 'route'=>implode(" ", $this->route )));
                    return;
                }

            }
        }

         if(     isset($found)   && isset($found['_function']) ){
            call_user_func_array( $found['_function'] , $found['_params'] );
        }
    }

    private function parseCommandParams($route=null){

        if( !isset($route) ){
            global $argv;
            array_shift($argv);
            $route = $argv;
        }else{
            $route = explode(" ",$route);
            $stop = true;
        }

        $this->route = $route;

        /* parse command line */
        foreach($route as $a){
            $a = trim($a);
            if(strpos($a,"--") === 0 ){
                $a = substr(trim($a),2);
                if(strpos($a, "=") !== false){
                    $ao = explode("=", $a);
                    $this->options[$ao[0]] = isset($ao[1]) ? trim($ao[1]) : true;
                }else{
                    $this->options[$a] = true;
                }
            }else{
                $this->commands[] = $a;
            }
        }

        if( isset($stop) ){
            exit();
        }

    }

    private function buildTree(){
        /* building trees */
        $this->controllersTree = $this->explodeTree($this->controllers, " ");
    }

    /**
    * @author   Kevin van Zonneveld <kevin@vanzonneveld.net>
    * @author   Lachlan Donald
    * @author   Takkie
    * @copyright 2008 Kevin van Zonneveld (http://kevin.vanzonneveld.net)
    * @license   http://www.opensource.org/licenses/bsd-license.php New BSD Licence
    * @version   SVN: Release: $Id: explodeTree.inc.php 89 2008-09-05 20:52:48Z kevin $
    * @link   http://kevin.vanzonneveld.net/
    *
    * @param array   $array
    * @param string  $delimiter
    * @param boolean $baseval
    *
    * @return array
    */
    private function explodeTree($array, $delimiter = '_', $baseval = false)
    {
      if(!is_array($array)) return false;
      $splitRE   = '/' . preg_quote($delimiter, '/') . '/';
      $returnArr = array();
      foreach ($array as $key => $val) {
          // Get parent parts and the current leaf
          $parts  = preg_split($splitRE, $key, -1, PREG_SPLIT_NO_EMPTY);
          $leafPart = array_pop($parts);

          // Build parent structure
          // Might be slow for really deep and large structures
          $parentArr = &$returnArr;
          foreach ($parts as $part) {
              if (!isset($parentArr[$part])) {
                  $parentArr[$part] = array();
              } elseif (!is_array($parentArr[$part])) {
                  if ($baseval) {
                      $parentArr[$part] = array('__base_val' => $parentArr[$part]);
                  } else {
                      $parentArr[$part] = array();
                  }
              }
              $parentArr = &$parentArr[$part];
          }

          // Add the final part to the structure
          if (empty($parentArr[$leafPart])) {
              $parentArr[$leafPart] = $val;
          } elseif ($baseval && is_array($parentArr[$leafPart])) {
              $parentArr[$leafPart]['__base_val'] = $val;
          }
      }
      return $returnArr;
    }

}


text:
<!DOCTYPE html>

  <meta charset="utf-8">

  <meta name="viewport" content="width=device-width, initial-scale=1.0">

  <title>MOAL.storage.modeling package — MoAL  documentation</title>

    <link rel="stylesheet" href="_static/css/theme.css" type="text/css" />

    <link rel="top" title="MoAL  documentation" href="index.html"/>
        <link rel="up" title="MOAL.storage package" href="MOAL.storage.html"/>
        <link rel="next" title="MOAL.storage.search package" href="MOAL.storage.search.html"/>
        <link rel="prev" title="MOAL.storage.databases.relational package" href="MOAL.storage.databases.relational.html"/>

             MoAL

  <form id="rtd-search-form" class="wy-form" action="search.html" method="get">
    <input type="text" name="q" placeholder="Search docs" />
    <input type="hidden" name="check_keywords" value="yes" />
    <input type="hidden" name="area" value="default" />
  </form>

MOAL package

Subpackages

MOAL.algorithms package

MOAL.artificial_intelligence package

MOAL.automata_theory package

MOAL.computer_organization package

MOAL.data_structures package

MOAL.execution package

MOAL.helpers package

MOAL.languages package

MOAL.maths package

MOAL.operating_systems package

MOAL.software_engineering package

MOAL.storage package

Subpackages

MOAL.systems_engineering package

MOAL.type_theory package

Submodules

MOAL.get_file_tree module

MOAL.modulify module

MOAL.test_files module

        MoAL

Docs »

MOAL package »

MOAL.storage package »

MOAL.storage.modeling package

             View page source

MOAL.storage.modeling package¶

Submodules¶

MOAL.storage.modeling.data_modeling module¶

MOAL.storage.modeling.data_modeling.DEBUG = False¶

From https://www.coursera.org/learn/data-manipulation/home/welcome
A Data Model is defined as having three primary properties:

Structure

rows and columns
nodes and edges
key-value pairs
sequence of bytes (raw, block data, file data)

Constraints

“all rows must have the same number of columns”
“all values in one column must have the same type”
“a child cannot have two parents”

Operations

“find value of key x”
“find rows where column lastname is jordan”
“get the next N bytes”

class MOAL.storage.modeling.data_modeling.Structure(*args, **kwargs)[source]¶

This is a somewhat arbitrary and meaningless class, but it
signifies some of these concepts in real code.

__getitem__(key)[source]¶

__init__(*args, **kwargs)[source]¶

constraint(constraint_name, field)[source]¶

operation(operation_name, val)[source]¶

show_model()[source]¶

MOAL.storage.modeling.data_modeling.create_field(schema, field, value)[source]¶

MOAL.storage.modeling.data_modeling.delete_field(schema, field)[source]¶

MOAL.storage.modeling.data_modeling.greater_than(val1, val2)[source]¶

MOAL.storage.modeling.data_modeling.less_than(val1, val2)[source]¶

MOAL.storage.modeling.data_modeling.longer(val1, val2)[source]¶

MOAL.storage.modeling.data_modeling.read_field(schema, field)[source]¶

MOAL.storage.modeling.data_modeling.shorter(val1, val2)[source]¶

MOAL.storage.modeling.data_modeling.update_field(schema, field, replace)[source]¶

        Next

         Previous

        © Copyright 2015, Chris Tabor.

  Built with Sphinx using a theme provided by Read the Docs.


text:
import * as Mongoose from 'mongoose';

export interface MongooseMeta {
  options: Mongoose.SchemaOptions; // options at the schema level
  schemaObj: Mongoose.SchemaDefinition; // schema definition of each path/property
  statics: { [key: string]: Function }; // used for static properties & functions
  methods: { [key: string]: Function }; // used for instance methods
  virtuals: { [key: string]: PropertyDescriptor }; // used for getter, setter functions & virtual ref (object: options inputs from decorators)
  // queries: { [key: string]: Function }; // use statics instead of query
}

export function hasMetadata(target: any): boolean {
  if (target.hasOwnProperty('__mongoose_meta__')) {
    const meta = getMetadata(target);
    return Object.keys(meta.schemaObj).length > 0;
  }
  return false;
}

export function getMetadata(target: any): MongooseMeta {
  if (!target.hasOwnProperty('__mongoose_meta__')) {
    // console.info(`Defining MongooseMeta for ${target.name}`);
    target.__mongoose_meta__ = {
      options: {},
      schemaObj: {},
      statics: {},
      // queries: {},
      methods: {},
      virtuals: {},
    };
  }
  return target.__mongoose_meta__ as MongooseMeta;
}


text:
<?php

namespace craft\events;

use yii\base\Event;

/**
 * SetAssetFilenameEvent class.
 *
 * @author Pixel & Tonic, Inc. <support@pixelandtonic.com>
 * @since 3.0
 */
class SetAssetFilenameEvent extends Event
{
    // Properties
    // =========================================================================

    /**
     * @var string|null The asset filename (sans extension).
     */
    public $filename;
}


text:
The `data` directory contains discovered MBRs of all forms and fashions.
If you happen to come across a MBR that hasn't been recorded here,
please send a Pull Request to add it!

Especially interesting would be samples of the following formats:

- AAP (Advanced Active Partitions)
- AST (AST Research / NEC MS-DOS and SpeedStor)
- DM (Disk Manager)
- NEWLDR

### Files

- `bootcamp-hybrid.bin` - Mac OS Bootcamp with GPT / MBR Hybdrid
- `rpi.bin` - Raspberry Pi
- `rpi3.bin` - Raspberry Pi 3
- `rufus.bin` - Rufus MBR
- `syslinux.bin` - SysLinux MBR


text:
package org.traccar.web.shared.model;

import com.google.gwt.user.client.rpc.IsSerializable;

public enum DeviceIconType implements IsSerializable {
    DEFAULT(PositionIconType.iconLatest, PositionIconType.iconOffline),
    PHONE(PositionIconType.phoneLatest, PositionIconType.phoneOffline),
    BICYCLE(PositionIconType.bicycleLatest, PositionIconType.bicycleOffline),
    SEDAN(PositionIconType.sedanLatest, PositionIconType.sedanOffline),
    UNIVERSAL(PositionIconType.universalLatest, PositionIconType.universalOffline),

    MINIVAN(PositionIconType.minivanLatest, PositionIconType.minivanOffline),
    TRUCK(PositionIconType.truckLatest, PositionIconType.truckOffline),
    BUS(PositionIconType.busLatest, PositionIconType.busOffline),
    LONG_TRUCK(PositionIconType.longTruckLatest, PositionIconType.longTruckOffline),
    CAR_TRUCK(PositionIconType.carTruckLatest, PositionIconType.carTruckOffline),

    PLANE(PositionIconType.planeLatest, PositionIconType.planeOffline),
    SHIP(PositionIconType.shipLatest, PositionIconType.shipOffline),
    TRAIN(PositionIconType.trainLatest, PositionIconType.trainOffline);

    private final PositionIconType iconLatest;
    private final PositionIconType iconOffline;

    DeviceIconType(PositionIconType iconLatest, PositionIconType iconOffline) {
        this.iconLatest = iconLatest;
        this.iconOffline = iconOffline;
    }

    public PositionIconType getPositionIconType(Position.Status status) {
        switch (status) {
            case LATEST:
                return iconLatest;
            case OFFLINE:
                return iconOffline;
        }
        return null;
    }
}


text:
The grid *animate* property is used to activate a slide animation for a [layout block](grid-layouts/drag-and-drop.md). The animation applies to the block itself and all associated block items.

?> When discussing grid layouts *content-within-blocks* are referred to as *block items*.

### Animate Activation

The following basic syntax is used to activate a *animation* behavior on a layout block:

```
[drag=width height, drop=x y, animate=type _speed]
```

Where the `type` option on the `animate=` property can take one of the following supported animation types:

- slideup
- slidedown
- slideleft
- slideright
- bounceup
- bouncedown
- bounceleft
- bounceright
- speedleft
- speedright
- tada
- flip
- fadein
- fadeout

?> The `_speed` option is optional. When specified it accepts a value of *slower* or *faster*.

Here is a short video clip that demonstrates block animation in action:

![Sample slide demonstrating grid layouts animations](../_images/gitpitch-grid-layouts-animations.gif)


text:
"""Base Driver for DataCore SANsymphony storage array."""

import math
import time

from oslo_config import cfg
from oslo_log import log as logging
from oslo_service import loopingcall
from oslo_utils import excutils
from oslo_utils import units

from cinder import context as cinder_context
from cinder import exception as cinder_exception
from cinder.i18n import _
from cinder import interface
from cinder import utils as cinder_utils
from cinder.volume import configuration
from cinder.volume import driver
from cinder.volume.drivers.datacore import api
from cinder.volume.drivers.datacore import exception as datacore_exception
from cinder.volume.drivers.datacore import utils as datacore_utils
from cinder.volume.drivers.san import san
from cinder.volume import volume_types

LOG = logging.getLogger(__name__)

datacore_opts = [
    cfg.StrOpt('datacore_disk_type',
               default='single',
               choices=['single', 'mirrored'],
               help='DataCore virtual disk type (single/mirrored). '
                    'Mirrored virtual disks require two storage servers in '
                    'the server group.'),
    cfg.StrOpt('datacore_storage_profile',
               default=None,
               help='DataCore virtual disk storage profile.'),
    cfg.ListOpt('datacore_disk_pools',
                default=[],
                help='List of DataCore disk pools that can be used '
                     'by volume driver.'),
    cfg.IntOpt('datacore_api_timeout',
               default=300,
               min=1,
               help='Seconds to wait for a response from a '
                    'DataCore API call.'),
    cfg.IntOpt('datacore_disk_failed_delay',
               default=300,
               min=0,
               help='Seconds to wait for DataCore virtual '
                    'disk to come out of the "Failed" state.'),
]

CONF = cfg.CONF
CONF.register_opts(datacore_opts, group=configuration.SHARED_CONF_GROUP)

@interface.volumedriver
class DataCoreVolumeDriver(driver.VolumeDriver):
    """DataCore SANsymphony base volume driver."""

    STORAGE_PROTOCOL = 'N/A'

    AWAIT_DISK_ONLINE_INTERVAL = 10
    AWAIT_SNAPSHOT_ONLINE_INTERVAL = 10
    AWAIT_SNAPSHOT_ONLINE_INITIAL_DELAY = 5

    DATACORE_SINGLE_DISK = 'single'
    DATACORE_MIRRORED_DISK = 'mirrored'
    VOLUME_TYPE_STRIPED = 'Striped'
    VOLUME_TYPE_SPANNED = 'Spanned'

    DATACORE_DISK_TYPE_KEY = 'datacore:disk_type'
    DATACORE_STORAGE_PROFILE_KEY = 'datacore:storage_profile'
    DATACORE_DISK_POOLS_KEY = 'datacore:disk_pools'

    VALID_VOLUME_TYPE_KEYS = (DATACORE_DISK_TYPE_KEY,
                              DATACORE_STORAGE_PROFILE_KEY,
                              DATACORE_DISK_POOLS_KEY,)

    def __init__(self, *args, **kwargs):
        super(DataCoreVolumeDriver, self).__init__(*args, **kwargs)
        self.configuration.append_config_values(san.san_opts)
        self.configuration.append_config_values(datacore_opts)
        self._api = None
        self._default_volume_options = None

    def do_setup(self, context):
        """Perform validations and establish connection to server.

        :param context: Context information
        """

        required_params = [
            'san_ip',
            'san_login',
            'san_password',
        ]
        for param in required_params:
            if not getattr(self.configuration, param, None):
                raise cinder_exception.InvalidInput(_("%s not set.") % param)

        self._api = api.DataCoreClient(
            self.configuration.san_ip,
            self.configuration.san_login,
            self.configuration.san_password,
            self.configuration.datacore_api_timeout)

        disk_type = self.configuration.datacore_disk_type
        if disk_type:
            disk_type = disk_type.lower()
        storage_profile = self.configuration.datacore_storage_profile
        if storage_profile:
            storage_profile = storage_profile.lower()
        disk_pools = self.configuration.datacore_disk_pools
        if disk_pools:
            disk_pools = [pool.lower() for pool in disk_pools]

        self._default_volume_options = {
            self.DATACORE_DISK_TYPE_KEY: disk_type,
            self.DATACORE_STORAGE_PROFILE_KEY: storage_profile,
            self.DATACORE_DISK_POOLS_KEY: disk_pools,
        }

    def check_for_setup_error(self):
        pass

    def get_volume_backend_name(self):
        """Get volume backend name of the volume service.

        :return: Volume backend name
        """

        backend_name = self.configuration.safe_get('volume_backend_name')
        return (backend_name or 'DataCore' + self.__class__.__name__)

    def create_volume(self, volume):
        """Creates a volume.

        :param volume: Volume object
        :return: Dictionary of changes to the volume object to be persisted
        """

        volume_options = self._get_volume_options(volume)

        disk_type = volume_options[self.DATACORE_DISK_TYPE_KEY]
        if disk_type == self.DATACORE_MIRRORED_DISK:
            logical_disk_count = 2
            virtual_disk_type = 'MultiPathMirrored'
        elif disk_type == self.DATACORE_SINGLE_DISK:
            logical_disk_count = 1
            virtual_disk_type = 'NonMirrored'
        else:
            msg = _("Virtual disk type '%s' is not valid.") % disk_type
            LOG.error(msg)
            raise cinder_exception.VolumeDriverException(message=msg)

        profile_id = self._get_storage_profile_id(
            volume_options[self.DATACORE_STORAGE_PROFILE_KEY])

        pools = datacore_utils.get_distinct_by(
            lambda pool: pool.ServerId,
            self._get_available_disk_pools(
                volume_options[self.DATACORE_DISK_POOLS_KEY]))

        if len(pools) < logical_disk_count:
            msg = _("Suitable disk pools were not found for "
                    "creating virtual disk.")
            LOG.error(msg)
            raise cinder_exception.VolumeDriverException(message=msg)

        disk_size = self._get_size_in_bytes(volume.size)

        logical_disks = []
        virtual_disk = None
        try:
            for logical_disk_pool in pools[:logical_disk_count]:
                logical_disks.append(
                    self._api.create_pool_logical_disk(
                        logical_disk_pool.Id,
                        self.VOLUME_TYPE_STRIPED, disk_size))

            virtual_disk_data = self._api.build_virtual_disk_data(
                volume.id,
                virtual_disk_type,
                disk_size,
                volume.display_name,
                profile_id)

            virtual_disk = self._api.create_virtual_disk_ex2(
                virtual_disk_data,
                logical_disks[0].Id,
                logical_disks[1].Id if logical_disk_count == 2 else None,
                True)

            virtual_disk = self._await_virtual_disk_online(virtual_disk.Id)

        except Exception:
            with excutils.save_and_reraise_exception():
                LOG.exception("Creation of volume %(volume)s failed.",
                              {'volume': volume.id})
                try:
                    if virtual_disk:
                        self._api.delete_virtual_disk(virtual_disk.Id, True)
                    else:
                        for logical_disk in logical_disks:
                            self._api.delete_logical_disk(logical_disk.Id)
                except datacore_exception.DataCoreException as e:
                    LOG.warning("An error occurred on a cleanup after failed "
                                "creation of volume %(volume)s: %(error)s.",
                                {'volume': volume.id, 'error': e})

        return {'provider_location': virtual_disk.Id}

    def create_volume_from_snapshot(self, volume, snapshot):
        """Creates a volume from a snapshot.

        :param volume: Volume object
        :param snapshot: Snapshot object
        :return: Dictionary of changes to the volume object to be persisted
        """
        return self._create_volume_from(volume, snapshot)

    def create_cloned_volume(self, volume, src_vref):
        """Creates volume clone.

        :param volume: New Volume object
        :param src_vref: Volume object that must be cloned
        :return: Dictionary of changes to the volume object to be persisted
        """

        return self._create_volume_from(volume, src_vref)

    def extend_volume(self, volume, new_size):
        """Extend an existing volume's size.

        :param volume: Volume object
        :param new_size: new size in GB to extend this volume to
        """

        virtual_disk = self._get_virtual_disk_for(volume, raise_not_found=True)
        self._set_virtual_disk_size(virtual_disk,
                                    self._get_size_in_bytes(new_size))
        virtual_disk = self._await_virtual_disk_online(virtual_disk.Id)

    def delete_volume(self, volume):
        """Deletes a volume.

        :param volume: Volume object
        """

        virtual_disk = self._get_virtual_disk_for(volume)
        if virtual_disk:
            if virtual_disk.IsServed:
                logical_disks = self._api.get_logical_disks()
                logical_units = self._api.get_logical_units()
                target_devices = self._api.get_target_devices()
                logical_disks = [disk.Id for disk in logical_disks
                                 if disk.VirtualDiskId == virtual_disk.Id]
                logical_unit_devices = [unit.VirtualTargetDeviceId
                                        for unit in logical_units
                                        if unit.LogicalDiskId in logical_disks]
                initiator_ports = set(device.InitiatorPortId
                                      for device in target_devices
                                      if device.Id in logical_unit_devices)
                for port in initiator_ports:
                    self._api.unserve_virtual_disks_from_port(
                        port, [virtual_disk.Id])
            self._api.delete_virtual_disk(virtual_disk.Id, True)

    def create_snapshot(self, snapshot):
        """Creates a snapshot.

        :param snapshot: Snapshot object
        :return: Dictionary of changes to the snapshot object to be persisted
        """

        src_virtual_disk = self._get_virtual_disk_for(snapshot.volume,
                                                      raise_not_found=True)

        volume_options = self._get_volume_options(snapshot.volume)
        profile_name = volume_options[self.DATACORE_STORAGE_PROFILE_KEY]
        profile_id = self._get_storage_profile_id(profile_name)
        pool_names = volume_options[self.DATACORE_DISK_POOLS_KEY]

        if src_virtual_disk.DiskStatus != 'Online':
            LOG.warning("Attempting to make a snapshot from virtual disk "
                        "%(disk)s that is in %(state)s state.",
                        {'disk': src_virtual_disk.Id,
                         'state': src_virtual_disk.DiskStatus})

        snapshot_virtual_disk = self._create_virtual_disk_copy(
            src_virtual_disk,
            snapshot.id,
            snapshot.display_name,
            profile_id=profile_id,
            pool_names=pool_names)
        snapshot_virtual_disk = self._await_virtual_disk_online(
            snapshot_virtual_disk.Id)

        return {'provider_location': snapshot_virtual_disk.Id}

    def delete_snapshot(self, snapshot):
        """Deletes a snapshot.

        :param snapshot: Snapshot object
        """

        snapshot_virtual_disk = self._get_virtual_disk_for(snapshot)
        if snapshot_virtual_disk:
            self._api.delete_virtual_disk(snapshot_virtual_disk.Id, True)

    def ensure_export(self, context, volume):
        pass

    def create_export(self, context, volume, connector):
        pass

    def remove_export(self, context, volume):
        pass

    def unserve_virtual_disks_from_host(self, volume, connector):
        virtual_disk = self._get_virtual_disk_for(volume)
        if virtual_disk:
            if connector is None:
                clients = self._api.get_clients()
            else:
                clients = [self._get_client(connector['host'],
                                            create_new=False)]

            server_group = self._get_our_server_group()

            @cinder_utils.synchronized(
                'datacore-backend-%s' % server_group.Id, external=True)
            def unserve_virtual_disk(client_id):
                self._api.unserve_virtual_disks_from_host(
                    client_id, [virtual_disk.Id])

            for client in clients:
                unserve_virtual_disk(client.Id)

    def terminate_connection(self, volume, connector, **kwargs):
        """Disallow connection from connector.

        :param volume: Volume object
        :param connector: Connector information
        """
        self.unserve_virtual_disks_from_host(volume, connector)

    def manage_existing(self, volume, existing_ref):
        return self.manage_existing_object(volume, existing_ref, "volume")

    def manage_existing_get_size(self, volume, existing_ref):
        return self.manage_existing_object_get_size(volume, existing_ref,
                                                    "volume")

    def manage_existing_snapshot(self, snapshot, existing_ref):
        return self.manage_existing_object(snapshot, existing_ref, "snapshot")

    def manage_existing_snapshot_get_size(self, snapshot, existing_ref):
        return self.manage_existing_object_get_size(snapshot, existing_ref,
                                                    "snapshot")

    def manage_existing_object(self, existing_object, existing_ref,
                               object_type):
        if 'source-name' not in existing_ref:
            reason = _('Reference must contain source-name element.')
            raise cinder_exception.ManageExistingInvalidReference(
                existing_ref=existing_ref, reason=reason)

        vd_alias = existing_ref['source-name']
        virtual_disk = datacore_utils.get_first_or_default(
            lambda disk: disk.Alias == vd_alias,
            self._api.get_virtual_disks(),
            None)

        if not virtual_disk:
            kwargs = {'existing_ref': vd_alias,
                      'reason': 'Specified Virtual disk does not exist.'}
            raise cinder_exception.ManageExistingInvalidReference(**kwargs)

        return {'provider_location': virtual_disk.Id}

    def manage_existing_object_get_size(self, existing_object, existing_ref,
                                        object_type):
        if 'source-name' not in existing_ref:
            reason = _('Reference must contain source-name element.')
            raise cinder_exception.ManageExistingInvalidReference(
                existing_ref=existing_ref, reason=reason)

        vd_alias = existing_ref['source-name']
        virtual_disk = datacore_utils.get_first_or_default(
            lambda disk: disk.Alias == vd_alias,
            self._api.get_virtual_disks(),
            None)

        if not virtual_disk:
            kwargs = {'existing_ref': vd_alias,
                      'reason': 'Specified Virtual disk does not exist.'}
            raise cinder_exception.ManageExistingInvalidReference(**kwargs)
        return(self._get_size_in_gigabytes(virtual_disk.Size.Value))

    def _update_volume_stats(self):
        performance_data = self._api.get_performance_by_type(
            ['DiskPoolPerformance'])
        total = 0
        available = 0
        reserved = 0
        for performance in performance_data:
            missing_perf_data = []

            if hasattr(performance.PerformanceData, 'BytesTotal'):
                total += performance.PerformanceData.BytesTotal
            else:
                missing_perf_data.append('BytesTotal')

            if hasattr(performance.PerformanceData, 'BytesAvailable'):
                available += performance.PerformanceData.BytesAvailable
            else:
                missing_perf_data.append('BytesAvailable')

            if hasattr(performance.PerformanceData, 'BytesReserved'):
                reserved += performance.PerformanceData.BytesReserved
            else:
                missing_perf_data.append('BytesReserved')

            if missing_perf_data:
                LOG.warning("Performance data %(data)s is missing for "
                            "disk pool %(pool)s",
                            {'data': missing_perf_data,
                             'pool': performance.ObjectId})
        provisioned = 0
        logical_disks = self._api.get_logical_disks()
        for disk in logical_disks:
            if getattr(disk, 'PoolId', None):
                provisioned += disk.Size.Value
        total_capacity_gb = self._get_size_in_gigabytes(total)
        free = available + reserved
        free_capacity_gb = self._get_size_in_gigabytes(free)
        provisioned_capacity_gb = self._get_size_in_gigabytes(provisioned)
        reserved_percentage = 100.0 * reserved / total if total else 0.0
        reserved_percentage = math.ceil(reserved_percentage)
        ratio = self.configuration.max_over_subscription_ratio
        stats_data = {
            'vendor_name': 'DataCore',
            'QoS_support': False,
            'volume_backend_name': self.get_volume_backend_name(),
            'driver_version': self.get_version(),
            'storage_protocol': self.STORAGE_PROTOCOL,
            'total_capacity_gb': total_capacity_gb,
            'free_capacity_gb': free_capacity_gb,
            'provisioned_capacity_gb': provisioned_capacity_gb,
            'reserved_percentage': reserved_percentage,
            'max_over_subscription_ratio': ratio,
            'thin_provisioning_support': True,
            'thick_provisioning_support': False,
            'online_extend_support': False,
        }
        self._stats = stats_data

    def _get_our_server_group(self):
        server_group = datacore_utils.get_first(lambda group: group.OurGroup,
                                                self._api.get_server_groups())

        return server_group

    def _get_volume_options_from_type(self, type_id, default_options):
        options = dict(default_options.items())
        if type_id:
            admin_context = cinder_context.get_admin_context()
            volume_type = volume_types.get_volume_type(admin_context, type_id)
            specs = dict(volume_type).get('extra_specs')

            for key, value in specs.items():
                if key in self.VALID_VOLUME_TYPE_KEYS:
                    if key == self.DATACORE_DISK_POOLS_KEY:
                        options[key] = [v.strip().lower()
                                        for v in value.split(',')]
                    else:
                        options[key] = value.lower()

        return options

    def _get_volume_options(self, volume):
        type_id = volume.volume_type_id

        volume_options = self._get_volume_options_from_type(
            type_id, self._default_volume_options)

        return volume_options

    def _get_online_servers(self):
        servers = self._api.get_servers()
        online_servers = [server for server in servers
                          if server.State == 'Online']
        return online_servers

    def _get_available_disk_pools(self, disk_pool_names=None):
        online_servers = [server.Id for server in self._get_online_servers()]

        pool_performance = {
            performance.ObjectId: performance.PerformanceData for performance
            in self._api.get_performance_by_type(['DiskPoolPerformance'])}

        disk_pools = self._api.get_disk_pools()

        lower_disk_pool_names = ([name.lower() for name in disk_pool_names]
                                 if disk_pool_names else [])

        available_disk_pools = [
            pool for pool in disk_pools
            if (self._is_pool_healthy(pool, pool_performance,
                                      online_servers) and
                (not lower_disk_pool_names or
                 pool.Caption.lower() in lower_disk_pool_names))]

        available_disk_pools.sort(
            key=lambda p: pool_performance[p.Id].BytesAvailable, reverse=True)

        return available_disk_pools

    def _get_virtual_disk_for(self, obj, raise_not_found=False):
        disk_id = obj.get('provider_location')

        virtual_disk = datacore_utils.get_first_or_default(
            lambda disk: disk.Id == disk_id,
            self._api.get_virtual_disks(),
            None)
        if not virtual_disk:
            msg = (_("Virtual disk not found for %(object)s %(object_id)s.")
                   % {'object': obj.__class__.__name__.lower(),
                      'object_id': obj['id']})
            if raise_not_found:
                LOG.error(msg)
                raise cinder_exception.VolumeDriverException(message=msg)
            else:
                LOG.warning(msg)

        return virtual_disk

    def _set_virtual_disk_size(self, virtual_disk, new_size):
        return self._api.set_virtual_disk_size(virtual_disk.Id, new_size)

    def _get_storage_profile(self, profile_name, raise_not_found=False):
        profiles = self._api.get_storage_profiles()
        profile = datacore_utils.get_first_or_default(
            lambda p: p.Caption.lower() == profile_name.lower(),
            profiles,
            None)
        if not profile and raise_not_found:
            msg = (_("Specified storage profile %s not found.")
                   % profile_name)
            LOG.error(msg)
            raise cinder_exception.VolumeDriverException(message=msg)

        return profile

    def _get_storage_profile_id(self, profile_name):
        profile_id = None
        if profile_name:
            profile = self._get_storage_profile(profile_name,
                                                raise_not_found=True)
            profile_id = profile.Id
        return profile_id

    def _await_virtual_disk_online(self, virtual_disk_id):
        def inner(start_time):
            disk_failed_delay = self.configuration.datacore_disk_failed_delay
            virtual_disk = datacore_utils.get_first(
                lambda disk: disk.Id == virtual_disk_id,
                self._api.get_virtual_disks())
            if virtual_disk.DiskStatus == 'Online':
                raise loopingcall.LoopingCallDone(virtual_disk)
            elif (
                    virtual_disk.DiskStatus != 'FailedRedundancy' and
                    time.time() - start_time >= disk_failed_delay):
                msg = (_("Virtual disk %(disk)s did not come out of the "
                         "%(state)s state after %(timeout)s seconds.")
                       % {'disk': virtual_disk.Id,
                          'state': virtual_disk.DiskStatus,
                          'timeout': disk_failed_delay})
                LOG.error(msg)
                raise cinder_exception.VolumeDriverException(message=msg)

        inner_loop = loopingcall.FixedIntervalLoopingCall(inner, time.time())
        return inner_loop.start(self.AWAIT_DISK_ONLINE_INTERVAL).wait()

    def _create_volume_from(self, volume, src_obj):
        src_virtual_disk = self._get_virtual_disk_for(src_obj,
                                                      raise_not_found=True)
        if src_virtual_disk.DiskStatus != 'Online':
            LOG.warning("Attempting to create a volume from virtual disk "
                        "%(disk)s that is in %(state)s state.",
                        {'disk': src_virtual_disk.Id,
                         'state': src_virtual_disk.DiskStatus})

        volume_options = self._get_volume_options(volume)
        profile_id = self._get_storage_profile_id(
            volume_options[self.DATACORE_STORAGE_PROFILE_KEY])
        pool_names = volume_options[self.DATACORE_DISK_POOLS_KEY]

        volume_virtual_disk = self._create_virtual_disk_copy(
            src_virtual_disk,
            volume.id,
            volume.display_name,
            profile_id=profile_id,
            pool_names=pool_names)

        volume_logical_disk = datacore_utils.get_first(
            lambda disk: disk.VirtualDiskId == volume_virtual_disk.Id,
            self._api.get_logical_disks())

        try:
            disk_type = volume_options[self.DATACORE_DISK_TYPE_KEY]
            if disk_type == self.DATACORE_MIRRORED_DISK:
                pools = self._get_available_disk_pools(pool_names)
                selected_pool = datacore_utils.get_first_or_default(
                    lambda pool: (pool.ServerId !=
                                  volume_logical_disk.ServerHostId and
                                  pool.Id != volume_logical_disk.PoolId),
                    pools, None)
                if selected_pool:
                    logical_disk = self._api.create_pool_logical_disk(
                        selected_pool.Id,
                        self.VOLUME_TYPE_STRIPED,
                        volume_virtual_disk.Size.Value)
                    self._api.bind_logical_disk(volume_virtual_disk.Id,
                                                logical_disk.Id,
                                                'Second',
                                                True,
                                                False,
                                                True)
                else:
                    msg = _("Can not create mirrored virtual disk. "
                            "Suitable disk pools not found.")
                    LOG.error(msg)
                    raise cinder_exception.VolumeDriverException(message=msg)
            volume_virtual_disk = self._await_virtual_disk_online(
                volume_virtual_disk.Id)
            try:
                source_size = src_obj.size
            except AttributeError:
                source_size = src_obj.volume_size
            if volume.size > source_size:
                self._set_virtual_disk_size(volume_virtual_disk,
                                            self._get_size_in_bytes(
                                                volume.size))
                volume_virtual_disk = datacore_utils.get_first(
                    lambda disk: disk.Id == volume_virtual_disk.Id,
                    self._api.get_virtual_disks())
                volume_virtual_disk = self._await_virtual_disk_size_change(
                    volume_virtual_disk.Id,
                    self._get_size_in_bytes(source_size))

        except Exception:
            with excutils.save_and_reraise_exception():
                LOG.exception("Creation of volume %(volume)s failed.",
                              {'volume': volume.id})
                try:
                    self._api.delete_virtual_disk(volume_virtual_disk.Id, True)
                except datacore_exception.DataCoreException as e:
                    LOG.warning("An error occurred on a cleanup after failed "
                                "creation of volume %(volume)s: %(error)s.",
                                {'volume': volume.id, 'error': e})

        return {'provider_location': volume_virtual_disk.Id}

    def _create_full_snapshot(self, description, name, pool_names, profile_id,
                              src_virtual_disk):
        pools = self._get_available_disk_pools(pool_names)
        destination_pool = datacore_utils.get_first_or_default(
            lambda pool: (pool.ServerId == src_virtual_disk.FirstHostId or
                          pool.ServerId == src_virtual_disk.SecondHostId),
            pools, None)

        if not destination_pool:
            msg = _("Suitable snapshot destination disk pool not found for "
                    "virtual disk %s.") % src_virtual_disk.Id
            LOG.error(msg)
            raise cinder_exception.VolumeDriverException(message=msg)
        server = datacore_utils.get_first(
            lambda srv: srv.Id == destination_pool.ServerId,
            self._api.get_servers())
        if not server.SnapshotMapStorePoolId:
            self._api.designate_map_store(destination_pool.Id)
        snapshot = self._api.create_snapshot(src_virtual_disk.Id,
                                             name,
                                             description,
                                             destination_pool.Id,
                                             'Full',
                                             False,
                                             profile_id)
        return snapshot

    def _await_snapshot_migrated(self, snapshot_id):
        def inner():
            snapshot_data = datacore_utils.get_first(
                lambda snapshot: snapshot.Id == snapshot_id,
                self._api.get_snapshots())
            if snapshot_data.State == 'Migrated':
                raise loopingcall.LoopingCallDone(snapshot_data)
            elif (snapshot_data.State != 'Healthy' and
                  snapshot_data.Failure != 'NoFailure'):
                msg = (_("Full migration of snapshot %(snapshot)s failed. "
                         "Snapshot is in %(state)s state.")
                       % {'snapshot': snapshot_data.Id,
                          'state': snapshot_data.State})
                LOG.error(msg)
                raise cinder_exception.VolumeDriverException(message=msg)

        loop = loopingcall.FixedIntervalLoopingCall(inner)
        time.sleep(self.AWAIT_SNAPSHOT_ONLINE_INTERVAL)
        return loop.start(self.AWAIT_SNAPSHOT_ONLINE_INTERVAL,
                          self.AWAIT_SNAPSHOT_ONLINE_INITIAL_DELAY).wait()

    def _create_virtual_disk_copy(self, src_virtual_disk, name, description,
                                  profile_id=None, pool_names=None):
        snapshot = self._create_full_snapshot(
            description, name, pool_names, profile_id, src_virtual_disk)

        try:
            snapshot = self._await_snapshot_migrated(snapshot.Id)
            self._api.delete_snapshot(snapshot.Id)
            self._await_snapshot_split_state_change(snapshot)
        except Exception:
            with excutils.save_and_reraise_exception():
                LOG.exception("Split operation failed for snapshot "
                              "%(snapshot)s.", {'snapshot': snapshot.Id})
                try:
                    logical_disk_copy = datacore_utils.get_first(
                        lambda disk: (disk.Id ==
                                      snapshot.DestinationLogicalDiskId),
                        self._api.get_logical_disks())

                    virtual_disk_copy = datacore_utils.get_first(
                        lambda disk: (disk.Id ==
                                      logical_disk_copy.VirtualDiskId),
                        self._api.get_virtual_disks())

                    self._api.delete_virtual_disk(virtual_disk_copy.Id, True)
                except datacore_exception.DataCoreException as e:
                    LOG.warning("An error occurred on a cleanup after failed "
                                "split of snapshot %(snapshot)s: %(error)s.",
                                {'snapshot': snapshot.Id, 'error': e})

        logical_disk_copy = datacore_utils.get_first(
            lambda disk: disk.Id == snapshot.DestinationLogicalDiskId,
            self._api.get_logical_disks())

        virtual_disk_copy = datacore_utils.get_first(
            lambda disk: disk.Id == logical_disk_copy.VirtualDiskId,
            self._api.get_virtual_disks())

        return virtual_disk_copy

    def _get_client(self, name, create_new=False):
        client_hosts = self._api.get_clients()

        client = datacore_utils.get_first_or_default(
            lambda host: host.HostName.split('.')[0] == name.split('.')[0],
            client_hosts, None)

        if create_new:
            if not client:
                client = self._api.register_client(
                    name, None, 'Other', 'PreferredServer', None)
            self._api.set_client_capabilities(client.Id, True, True)

        return client

    @staticmethod
    def _is_pool_healthy(pool, pool_performance, online_servers):
        if (pool.PoolStatus == 'Running' and
                hasattr(pool_performance[pool.Id], 'BytesAvailable') and
                pool.ServerId in online_servers):
            return True
        return False

    @staticmethod
    def _get_size_in_bytes(size_in_gigabytes):
        return size_in_gigabytes * units.Gi

    @staticmethod
    def _get_size_in_gigabytes(size_in_bytes):
        return size_in_bytes / float(units.Gi)

    def _await_virtual_disk_size_change(self, virtual_disk_id, old_size):
        def inner(start_time):
            disk_failed_delay = self.configuration.datacore_disk_failed_delay
            virtual_disk = datacore_utils.get_first(
                lambda disk: disk.Id == virtual_disk_id,
                self._api.get_virtual_disks())
            if virtual_disk.DiskStatus == 'Online' \
                    and virtual_disk.Size.Value > old_size:
                raise loopingcall.LoopingCallDone(virtual_disk)
            elif (virtual_disk.DiskStatus != 'FailedRedundancy' and
                  time.time() - start_time >= disk_failed_delay):
                msg = (_("Virtual disk %(disk)s did not come out of the "
                         "%(state)s state after %(timeout)s seconds.")
                       % {'disk': virtual_disk.Id,
                          'state': virtual_disk.DiskStatus,
                          'timeout': disk_failed_delay})
                LOG.error(msg)
                raise cinder_exception.VolumeDriverException(message=msg)

        inner_loop = loopingcall.FixedIntervalLoopingCall(inner, time.time())
        time.sleep(self.AWAIT_DISK_ONLINE_INTERVAL)
        return inner_loop.start(self.AWAIT_DISK_ONLINE_INTERVAL).wait()

    def _await_snapshot_split_state_change(self, split_snapshot):
        def inner(start_time):
            disk_failed_delay = self.configuration.datacore_disk_failed_delay
            snapshot_found = False
            snapshot_list = self._api.get_snapshots()
            if not snapshot_list:
                raise loopingcall.LoopingCallDone()
            for entry in snapshot_list:
                if entry.Caption == split_snapshot.Caption:
                    snapshot_found = True
                    break
            if not snapshot_found:
                raise loopingcall.LoopingCallDone()
            if (time.time() - start_time >= disk_failed_delay):
                msg = (_("Split Snapshot disk %(disk)s did not happened "
                         "after %(timeout)s seconds.")
                       % {'disk': split_snapshot.Caption,
                          'timeout': disk_failed_delay})
                LOG.error(msg)
                raise loopingcall.LoopingCallDone()

        inner_loop = loopingcall.FixedIntervalLoopingCall(inner, time.time())
        return inner_loop.start(self.AWAIT_DISK_ONLINE_INTERVAL).wait()


text:
from __future__ import print_function

def parse_args():
    import argparse
    parser = argparse.ArgumentParser()

    parser.add_argument('-i', '--input',
                        required=True,
                        help='Input text file.')
    parser.add_argument('-m', '--map',
                        required=True,
                        help='Map file.')

    args = parser.parse_args()

    return args

def main():
    args = parse_args()
    mapping = {}
    with open(args.map, 'r') as f:
        for line in f:
            words = line.split()
            mapping[words[0]] = words[1]

    with open(args.input, 'r') as f:
        for line in f:
            words = line.split()
            words[1] = mapping[words[1]]
            print('\t'.join(words))

if __name__ == "__main__":
    main()


text:
<?xml version="1.0" encoding="ISO-8859-1"?>
<!--                                                                -->
<!--   W4F: Copyright Arnaud Sahuguet and Fabien Azavant, 1998-99   -->
<!--   URL: http://db.cis.upenn.edu/W4F                             -->
<!--                                                                -->
<!DOCTYPE W4F_DOC  SYSTEM "movies.dtd">
<W4F_DOC>
  <Movie>
    <Title>In the Heat of the Night</Title>
    <Year>1967</Year>
    <Directed_By>
      <Director>Norman Jewison</Director>
    </Directed_By>
    <Genres>
      <Genre>Drama</Genre>
      <Genre>Mystery</Genre>
      <Genre>(more)</Genre>
    </Genres>
    <Cast>
      <Actor>
        <FirstName>Sidney</FirstName>
        <LastName>Poitier</LastName>
      </Actor>
      <Actor>
        <FirstName>Rod</FirstName>
        <LastName>Steiger</LastName>
      </Actor>
      <Actor>
        <FirstName>Warren</FirstName>
        <LastName>Oates</LastName>
      </Actor>
      <Actor>
        <FirstName>Lee</FirstName>
        <LastName>Grant (I)</LastName>
      </Actor>
      <Actor>
        <FirstName>Larry</FirstName>
        <LastName>Gates</LastName>
      </Actor>
      <Actor>
        <FirstName>James</FirstName>
        <LastName>Patterson (I)</LastName>
      </Actor>
      <Actor>
        <FirstName>William</FirstName>
        <LastName>Schallert</LastName>
      </Actor>
      <Actor>
        <FirstName>Beah</FirstName>
        <LastName>Richards</LastName>
      </Actor>
      <Actor>
        <FirstName>Peter</FirstName>
        <LastName>Whitney</LastName>
      </Actor>
      <Actor>
        <FirstName>Kermit</FirstName>
        <LastName>Murdock</LastName>
      </Actor>
      <Actor>
        <FirstName>Larry</FirstName>
        <LastName>D. Mann</LastName>
      </Actor>
      <Actor>
        <FirstName>Matt</FirstName>
        <LastName>Clark (I)</LastName>
      </Actor>
      <Actor>
        <FirstName>Arthur</FirstName>
        <LastName>Malet</LastName>
      </Actor>
      <Actor>
        <FirstName>Fred</FirstName>
        <LastName>Stewart</LastName>
      </Actor>
      <Actor>
        <FirstName>Quentin</FirstName>
        <LastName>Dean</LastName>
      </Actor>
    </Cast>
  </Movie>
</W4F_DOC>