text:
package org.apache.cassandra.io.sstable;

import java.io.File;
import java.io.FilenameFilter;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.util.*;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.atomic.AtomicInteger;

import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import com.google.common.io.Files;

import org.junit.Before;
import org.junit.BeforeClass;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.TemporaryFolder;

import org.apache.cassandra.SchemaLoader;
import org.apache.cassandra.Util;
import org.apache.cassandra.config.*;
import org.apache.cassandra.cql3.*;
import org.apache.cassandra.cql3.functions.UDHelper;
import org.apache.cassandra.cql3.functions.types.*;
import org.apache.cassandra.db.Keyspace;
import org.apache.cassandra.db.commitlog.CommitLog;
import org.apache.cassandra.dht.*;
import org.apache.cassandra.exceptions.*;
import org.apache.cassandra.schema.Schema;
import org.apache.cassandra.schema.TableMetadataRef;
import org.apache.cassandra.service.StorageService;
import org.apache.cassandra.transport.ProtocolVersion;
import org.apache.cassandra.utils.*;

import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.fail;

public class CQLSSTableWriterTest
{
    private static final AtomicInteger idGen = new AtomicInteger(0);
    private String keyspace;
    private String table;
    private String qualifiedTable;
    private File dataDir;

    static
    {
        DatabaseDescriptor.daemonInitialization();
    }

    @Rule
    public TemporaryFolder tempFolder = new TemporaryFolder();

    @BeforeClass
    public static void setup() throws Exception
    {
        CommitLog.instance.start();
        SchemaLoader.cleanupAndLeaveDirs();
        Keyspace.setInitialized();
        StorageService.instance.initServer();
    }

    @Before
    public void perTestSetup() throws IOException
    {
        keyspace = "cql_keyspace" + idGen.incrementAndGet();
        table = "table" + idGen.incrementAndGet();
        qualifiedTable = keyspace + '.' + table;
        dataDir = new File(tempFolder.newFolder().getAbsolutePath() + File.separator + keyspace + File.separator + table);
        assert dataDir.mkdirs();
    }

    @Test
    public void testUnsortedWriter() throws Exception
    {
        try (AutoCloseable switcher = Util.switchPartitioner(ByteOrderedPartitioner.instance))
        {
            String schema = "CREATE TABLE " + qualifiedTable + " ("
                          + "  k int PRIMARY KEY,"
                          + "  v1 text,"
                          + "  v2 int"
                          + ")";
            String insert = "INSERT INTO " + qualifiedTable + " (k, v1, v2) VALUES (?, ?, ?)";
            CQLSSTableWriter writer = CQLSSTableWriter.builder()
                                                      .inDirectory(dataDir)
                                                      .forTable(schema)
                                                      .using(insert).build();

            writer.addRow(0, "test1", 24);
            writer.addRow(1, "test2", 44);
            writer.addRow(2, "test3", 42);
            writer.addRow(ImmutableMap.<String, Object>of("k", 3, "v2", 12));

            writer.close();

            loadSSTables(dataDir, keyspace);

            UntypedResultSet rs = QueryProcessor.executeInternal("SELECT * FROM " + qualifiedTable + ";");
            assertEquals(4, rs.size());

            Iterator<UntypedResultSet.Row> iter = rs.iterator();
            UntypedResultSet.Row row;

            row = iter.next();
            assertEquals(0, row.getInt("k"));
            assertEquals("test1", row.getString("v1"));
            assertEquals(24, row.getInt("v2"));

            row = iter.next();
            assertEquals(1, row.getInt("k"));
            assertEquals("test2", row.getString("v1"));
            //assertFalse(row.has("v2"));
            assertEquals(44, row.getInt("v2"));

            row = iter.next();
            assertEquals(2, row.getInt("k"));
            assertEquals("test3", row.getString("v1"));
            assertEquals(42, row.getInt("v2"));

            row = iter.next();
            assertEquals(3, row.getInt("k"));
            assertEquals(null, row.getBytes("v1")); // Using getBytes because we know it won't NPE
            assertEquals(12, row.getInt("v2"));
        }
    }

    @Test
    public void testForbidCounterUpdates() throws Exception
    {
        String schema = "CREATE TABLE " + qualifiedTable + " (" +
                        "  my_id int, " +
                        "  my_counter counter, " +
                        "  PRIMARY KEY (my_id)" +
                        ")";
        String insert = String.format("UPDATE " + qualifiedTable + " SET my_counter = my_counter - ? WHERE my_id = ?");
        try
        {
            CQLSSTableWriter.builder().inDirectory(dataDir)
                            .forTable(schema)
                            .withPartitioner(Murmur3Partitioner.instance)
                            .using(insert).build();
            fail("Counter update statements should not be supported");
        }
        catch (IllegalArgumentException e)
        {
            assertEquals(e.getMessage(), "Counter update statements are not supported");
        }
    }

    @Test
    public void testSyncWithinPartition() throws Exception
    {
        // Check that the write respect the buffer size even if we only insert rows withing the same partition (#7360)
        // To do that simply, we use a writer with a buffer of 1MB, and write 2 rows in the same partition with a value
        // > 1MB and validate that this created more than 1 sstable.
        String schema = "CREATE TABLE " + qualifiedTable + " ("
                      + "  k int PRIMARY KEY,"
                      + "  v blob"
                      + ")";
        String insert = "INSERT INTO " + qualifiedTable + " (k, v) VALUES (?, ?)";
        CQLSSTableWriter writer = CQLSSTableWriter.builder()
                                                  .inDirectory(dataDir)
                                                  .using(insert)
                                                  .forTable(schema)
                                                  .withBufferSizeInMB(1)
                                                  .build();

        ByteBuffer val = ByteBuffer.allocate(1024 * 1050);

        writer.addRow(0, val);
        writer.addRow(1, val);
        writer.close();

        FilenameFilter filterDataFiles = new FilenameFilter()
        {
            public boolean accept(File dir, String name)
            {
                return name.endsWith("-Data.db");
            }
        };
        assert dataDir.list(filterDataFiles).length > 1 : Arrays.toString(dataDir.list(filterDataFiles));
    }

    @Test
    public void testSyncNoEmptyRows() throws Exception
    {
        // Check that the write does not throw an empty partition error (#9071)
        String schema = "CREATE TABLE " + qualifiedTable + " ("
                        + "  k UUID,"
                        + "  c int,"
                        + "  PRIMARY KEY (k)"
                        + ")";
        String insert = "INSERT INTO " + qualifiedTable + " (k, c) VALUES (?, ?)";
        CQLSSTableWriter writer = CQLSSTableWriter.builder()
                                                  .inDirectory(dataDir)
                                                  .forTable(schema)
                                                  .using(insert)
                                                  .withBufferSizeInMB(1)
                                                  .build();

        for (int i = 0 ; i < 50000 ; i++) {
            writer.addRow(UUID.randomUUID(), 0);
        }
        writer.close();

    }

    private static final int NUMBER_WRITES_IN_RUNNABLE = 10;
    private class WriterThread extends Thread
    {
        private final File dataDir;
        private final int id;
        private final String qualifiedTable;
        public volatile Exception exception;

        public WriterThread(File dataDir, int id, String qualifiedTable)
        {
            this.dataDir = dataDir;
            this.id = id;
            this.qualifiedTable = qualifiedTable;
        }

        @Override
        public void run()
        {
            String schema = "CREATE TABLE " + qualifiedTable + " ("
                    + "  k int,"
                    + "  v int,"
                    + "  PRIMARY KEY (k, v)"
                    + ")";
            String insert = "INSERT INTO " + qualifiedTable + " (k, v) VALUES (?, ?)";
            CQLSSTableWriter writer = CQLSSTableWriter.builder()
                    .inDirectory(dataDir)
                    .forTable(schema)
                    .using(insert).build();

            try
            {
                for (int i = 0; i < NUMBER_WRITES_IN_RUNNABLE; i++)
                {
                    writer.addRow(id, i);
                }
                writer.close();
            }
            catch (Exception e)
            {
                exception = e;
            }
        }
    }

    @Test
    public void testConcurrentWriters() throws Exception
    {
        WriterThread[] threads = new WriterThread[5];
        for (int i = 0; i < threads.length; i++)
        {
            WriterThread thread = new WriterThread(dataDir, i, qualifiedTable);
            threads[i] = thread;
            thread.start();
        }

        for (WriterThread thread : threads)
        {
            thread.join();
            assert !thread.isAlive() : "Thread should be dead by now";
            if (thread.exception != null)
            {
                throw thread.exception;
            }
        }

        loadSSTables(dataDir, keyspace);

        UntypedResultSet rs = QueryProcessor.executeInternal("SELECT * FROM " + qualifiedTable + ";");
        assertEquals(threads.length * NUMBER_WRITES_IN_RUNNABLE, rs.size());
    }

    @Test
    @SuppressWarnings("unchecked")
    public void testWritesWithUdts() throws Exception
    {
        final String schema = "CREATE TABLE " + qualifiedTable + " ("
                              + "  k int,"
                              + "  v1 list<frozen<tuple2>>,"
                              + "  v2 frozen<tuple3>,"
                              + "  PRIMARY KEY (k)"
                              + ")";

        CQLSSTableWriter writer = CQLSSTableWriter.builder()
                                                  .inDirectory(dataDir)
                                                  .withType("CREATE TYPE " + keyspace + ".tuple2 (a int, b int)")
                                                  .withType("CREATE TYPE " + keyspace + ".tuple3 (a int, b int, c int)")
                                                  .forTable(schema)
                                                  .using("INSERT INTO " + keyspace + "." + table + " (k, v1, v2) " +
                                                         "VALUES (?, ?, ?)").build();

        UserType tuple2Type = writer.getUDType("tuple2");
        UserType tuple3Type = writer.getUDType("tuple3");
        for (int i = 0; i < 100; i++)
        {
            writer.addRow(i,
                          ImmutableList.builder()
                                       .add(tuple2Type.newValue()
                                                      .setInt("a", i * 10)
                                                      .setInt("b", i * 20))
                                       .add(tuple2Type.newValue()
                                                      .setInt("a", i * 30)
                                                      .setInt("b", i * 40))
                                       .build(),
                          tuple3Type.newValue()
                                    .setInt("a", i * 100)
                                    .setInt("b", i * 200)
                                    .setInt("c", i * 300));
        }

        writer.close();
        loadSSTables(dataDir, keyspace);

        UntypedResultSet resultSet = QueryProcessor.executeInternal("SELECT * FROM " + keyspace + "." + table);
        TypeCodec collectionCodec = UDHelper.codecFor(DataType.CollectionType.list(tuple2Type));
        TypeCodec tuple3Codec = UDHelper.codecFor(tuple3Type);

        assertEquals(resultSet.size(), 100);
        int cnt = 0;
        for (UntypedResultSet.Row row: resultSet) {
            assertEquals(cnt,
                         row.getInt("k"));
            List<UDTValue> values = (List<UDTValue>) collectionCodec.deserialize(row.getBytes("v1"),
                                                                                ProtocolVersion.CURRENT);
            assertEquals(values.get(0).getInt("a"), cnt * 10);
            assertEquals(values.get(0).getInt("b"), cnt * 20);
            assertEquals(values.get(1).getInt("a"), cnt * 30);
            assertEquals(values.get(1).getInt("b"), cnt * 40);

            UDTValue v2 = (UDTValue) tuple3Codec.deserialize(row.getBytes("v2"), ProtocolVersion.CURRENT);

            assertEquals(v2.getInt("a"), cnt * 100);
            assertEquals(v2.getInt("b"), cnt * 200);
            assertEquals(v2.getInt("c"), cnt * 300);
            cnt++;
        }
    }

    @Test
    @SuppressWarnings("unchecked")
    public void testWritesWithDependentUdts() throws Exception
    {
        final String schema = "CREATE TABLE " + qualifiedTable + " ("
                              + "  k int,"
                              + "  v1 frozen<nested_tuple>,"
                              + "  PRIMARY KEY (k)"
                              + ")";

        CQLSSTableWriter writer = CQLSSTableWriter.builder()
                                                  .inDirectory(dataDir)
                                                  .withType("CREATE TYPE " + keyspace + ".nested_tuple (c int, tpl frozen<tuple2>)")
                                                  .withType("CREATE TYPE " + keyspace + ".tuple2 (a int, b int)")
                                                  .forTable(schema)
                                                  .using("INSERT INTO " + keyspace + "." + table + " (k, v1) " +
                                                         "VALUES (?, ?)")
                                                  .build();

        UserType tuple2Type = writer.getUDType("tuple2");
        UserType nestedTuple = writer.getUDType("nested_tuple");
        TypeCodec tuple2Codec = UDHelper.codecFor(tuple2Type);
        TypeCodec nestedTupleCodec = UDHelper.codecFor(nestedTuple);

        for (int i = 0; i < 100; i++)
        {
            writer.addRow(i,
                          nestedTuple.newValue()
                                     .setInt("c", i * 100)
                                     .set("tpl",
                                          tuple2Type.newValue()
                                                    .setInt("a", i * 200)
                                                    .setInt("b", i * 300),
                                          tuple2Codec));
        }

        writer.close();
        loadSSTables(dataDir, keyspace);

        UntypedResultSet resultSet = QueryProcessor.executeInternal("SELECT * FROM " + keyspace + "." + table);

        assertEquals(resultSet.size(), 100);
        int cnt = 0;
        for (UntypedResultSet.Row row: resultSet) {
            assertEquals(cnt,
                         row.getInt("k"));
            UDTValue nestedTpl = (UDTValue) nestedTupleCodec.deserialize(row.getBytes("v1"),
                                                                         ProtocolVersion.CURRENT);
            assertEquals(nestedTpl.getInt("c"), cnt * 100);
            UDTValue tpl = nestedTpl.getUDTValue("tpl");
            assertEquals(tpl.getInt("a"), cnt * 200);
            assertEquals(tpl.getInt("b"), cnt * 300);

            cnt++;
        }
    }

    @Test
    public void testUnsetValues() throws Exception
    {
        final String schema = "CREATE TABLE " + qualifiedTable + " ("
                              + "  k int,"
                              + "  c1 int,"
                              + "  c2 int,"
                              + "  v text,"
                              + "  PRIMARY KEY (k, c1, c2)"
                              + ")";

        CQLSSTableWriter writer = CQLSSTableWriter.builder()
                                                  .inDirectory(dataDir)
                                                  .forTable(schema)
                                                  .using("INSERT INTO " + qualifiedTable + " (k, c1, c2, v) " +
                                                         "VALUES (?, ?, ?, ?)")
                                                  .build();

        try
        {
            writer.addRow(1, 1, 1);
            fail("Passing less arguments then expected in prepared statement should not work.");
        }
        catch (InvalidRequestException e)
        {
            assertEquals("Invalid number of arguments, expecting 4 values but got 3",
                         e.getMessage());
        }

        try
        {
            writer.addRow(1, 1, CQLSSTableWriter.UNSET_VALUE, "1");
            fail("Unset values should not work with clustering columns.");
        }
        catch (InvalidRequestException e)
        {
            assertEquals("Invalid unset value for column c2",
                         e.getMessage());
        }

        try
        {
            writer.addRow(ImmutableMap.<String, Object>builder().put("k", 1).put("c1", 1).put("v", CQLSSTableWriter.UNSET_VALUE).build());
            fail("Unset or null clustering columns should not be allowed.");
        }
        catch (InvalidRequestException e)
        {
            assertEquals("Invalid null value in condition for column c2",
                         e.getMessage());
        }

        writer.addRow(1, 1, 1, CQLSSTableWriter.UNSET_VALUE);
        writer.addRow(2, 2, 2, null);
        writer.addRow(Arrays.asList(3, 3, 3, CQLSSTableWriter.UNSET_VALUE));
        writer.addRow(ImmutableMap.<String, Object>builder()
                                  .put("k", 4)
                                  .put("c1", 4)
                                  .put("c2", 4)
                                  .put("v", CQLSSTableWriter.UNSET_VALUE)
                                  .build());
        writer.addRow(Arrays.asList(3, 3, 3, CQLSSTableWriter.UNSET_VALUE));
        writer.addRow(5, 5, 5, "5");

        writer.close();
        loadSSTables(dataDir, keyspace);

        UntypedResultSet resultSet = QueryProcessor.executeInternal("SELECT * FROM " + qualifiedTable);
        Iterator<UntypedResultSet.Row> iter = resultSet.iterator();
        UntypedResultSet.Row r1 = iter.next();
        assertEquals(1, r1.getInt("k"));
        assertEquals(1, r1.getInt("c1"));
        assertEquals(1, r1.getInt("c2"));
        assertEquals(false, r1.has("v"));
        UntypedResultSet.Row r2 = iter.next();
        assertEquals(2, r2.getInt("k"));
        assertEquals(2, r2.getInt("c1"));
        assertEquals(2, r2.getInt("c2"));
        assertEquals(false, r2.has("v"));
        UntypedResultSet.Row r3 = iter.next();
        assertEquals(3, r3.getInt("k"));
        assertEquals(3, r3.getInt("c1"));
        assertEquals(3, r3.getInt("c2"));
        assertEquals(false, r3.has("v"));
        UntypedResultSet.Row r4 = iter.next();
        assertEquals(4, r4.getInt("k"));
        assertEquals(4, r4.getInt("c1"));
        assertEquals(4, r4.getInt("c2"));
        assertEquals(false, r3.has("v"));
        UntypedResultSet.Row r5 = iter.next();
        assertEquals(5, r5.getInt("k"));
        assertEquals(5, r5.getInt("c1"));
        assertEquals(5, r5.getInt("c2"));
        assertEquals(true, r5.has("v"));
        assertEquals("5", r5.getString("v"));
    }

    @Test
    public void testUpdateStatement() throws Exception
    {
        final String schema = "CREATE TABLE " + qualifiedTable + " ("
                              + "  k int,"
                              + "  c1 int,"
                              + "  c2 int,"
                              + "  v text,"
                              + "  PRIMARY KEY (k, c1, c2)"
                              + ")";

        CQLSSTableWriter writer = CQLSSTableWriter.builder()
                                                  .inDirectory(dataDir)
                                                  .forTable(schema)
                                                  .using("UPDATE " + qualifiedTable + " SET v = ? " +
                                                         "WHERE k = ? AND c1 = ? AND c2 = ?")
                                                  .build();

        writer.addRow("a", 1, 2, 3);
        writer.addRow("b", 4, 5, 6);
        writer.addRow(null, 7, 8, 9);
        writer.addRow(CQLSSTableWriter.UNSET_VALUE, 10, 11, 12);
        writer.close();
        loadSSTables(dataDir, keyspace);

        UntypedResultSet resultSet = QueryProcessor.executeInternal("SELECT * FROM " + qualifiedTable);
        assertEquals(2, resultSet.size());

        Iterator<UntypedResultSet.Row> iter = resultSet.iterator();
        UntypedResultSet.Row r1 = iter.next();
        assertEquals(1, r1.getInt("k"));
        assertEquals(2, r1.getInt("c1"));
        assertEquals(3, r1.getInt("c2"));
        assertEquals("a", r1.getString("v"));
        UntypedResultSet.Row r2 = iter.next();
        assertEquals(4, r2.getInt("k"));
        assertEquals(5, r2.getInt("c1"));
        assertEquals(6, r2.getInt("c2"));
        assertEquals("b", r2.getString("v"));
        assertFalse(iter.hasNext());
    }

    @Test
    public void testNativeFunctions() throws Exception
    {
        final String schema = "CREATE TABLE " + qualifiedTable + " ("
                              + "  k int,"
                              + "  c1 int,"
                              + "  c2 int,"
                              + "  v blob,"
                              + "  PRIMARY KEY (k, c1, c2)"
                              + ")";

        CQLSSTableWriter writer = CQLSSTableWriter.builder()
                                                  .inDirectory(dataDir)
                                                  .forTable(schema)
                                                  .using("INSERT INTO " + qualifiedTable + " (k, c1, c2, v) VALUES (?, ?, ?, textAsBlob(?))")
                                                  .build();

        writer.addRow(1, 2, 3, "abc");
        writer.addRow(4, 5, 6, "efg");

        writer.close();
        loadSSTables(dataDir, keyspace);

        UntypedResultSet resultSet = QueryProcessor.executeInternal("SELECT * FROM " + qualifiedTable);
        assertEquals(2, resultSet.size());

        Iterator<UntypedResultSet.Row> iter = resultSet.iterator();
        UntypedResultSet.Row r1 = iter.next();
        assertEquals(1, r1.getInt("k"));
        assertEquals(2, r1.getInt("c1"));
        assertEquals(3, r1.getInt("c2"));
        assertEquals(ByteBufferUtil.bytes("abc"), r1.getBytes("v"));

        UntypedResultSet.Row r2 = iter.next();
        assertEquals(4, r2.getInt("k"));
        assertEquals(5, r2.getInt("c1"));
        assertEquals(6, r2.getInt("c2"));
        assertEquals(ByteBufferUtil.bytes("efg"), r2.getBytes("v"));

        assertFalse(iter.hasNext());
    }

    @Test
    public void testWriteWithNestedTupleUdt() throws Exception
    {
        // Check the writer does not throw "InvalidRequestException: Non-frozen tuples are not allowed inside collections: list<tuple<int, int>>"
        // See CASSANDRA-15857
        final String schema = "CREATE TABLE " + qualifiedTable + " ("
                              + "  k int,"
                              + "  v1 frozen<nested_type>,"
                              + "  PRIMARY KEY (k)"
                              + ")";

        CQLSSTableWriter writer = CQLSSTableWriter.builder()
                                                  .inDirectory(dataDir)
                                                  .withType("CREATE TYPE " + keyspace + ".nested_type (a list<tuple<int, int>>)")
                                                  .forTable(schema)
                                                  .using("INSERT INTO " + qualifiedTable + " (k, v1) " +
                                                         "VALUES (?, ?)").build();

        UserType nestedType = writer.getUDType("nested_type");
        for (int i = 0; i < 100; i++)
        {
            writer.addRow(i, nestedType.newValue()
                                       .setList("a", Collections.emptyList()));
        }

        writer.close();
        loadSSTables(dataDir, keyspace);

        UntypedResultSet resultSet = QueryProcessor.executeInternal("SELECT * FROM " + qualifiedTable);
        assertEquals(100, resultSet.size());
    }

    private static void loadSSTables(File dataDir, String ks) throws ExecutionException, InterruptedException
    {
        SSTableLoader loader = new SSTableLoader(dataDir, new SSTableLoader.Client()
        {
            private String keyspace;

            public void init(String keyspace)
            {
                this.keyspace = keyspace;
                for (Range<Token> range : StorageService.instance.getLocalReplicas(ks).ranges())
                    addRangeForEndpoint(range, FBUtilities.getBroadcastAddressAndPort());
            }

            public TableMetadataRef getTableMetadata(String cfName)
            {
                return Schema.instance.getTableMetadataRef(keyspace, cfName);
            }
        }, new OutputHandler.SystemOutput(false, false));

        loader.stream().get();
    }
}


text:
<?php

use Illuminate\Database\Migrations\Migration;
use Illuminate\Database\Schema\Blueprint;

class AddColumnUserIdToLinksTable extends Migration
{
    /**
     * Run the migrations.
     *
     * @return void
     */
    public function up()
    {
        Schema::table('links', function (Blueprint $table) {
            $table->integer('user_id')->nullable();
            $table->integer('project_id')->nullable();
        });
    }

    /**
     * Reverse the migrations.
     *
     * @return void
     */
    public function down()
    {
        Schema::table('links', function (Blueprint $table) {
            $table->dropColumn(['user_id', 'project_id']);
        });
    }
}

redpajama_set_name: RedPajamaGithub


text:
/*
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package com.facebook.presto.raptor.storage;

import com.facebook.presto.raptor.backup.BackupStore;
import com.facebook.presto.raptor.metadata.ShardManager;
import com.facebook.presto.raptor.metadata.ShardMetadata;
import com.facebook.presto.raptor.util.PrioritizedFifoExecutor;
import com.facebook.presto.spi.NodeManager;
import com.facebook.presto.spi.PrestoException;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.cache.CacheBuilder;
import com.google.common.cache.CacheLoader;
import com.google.common.cache.LoadingCache;
import com.google.common.util.concurrent.FutureCallback;
import com.google.common.util.concurrent.Futures;
import com.google.common.util.concurrent.ListenableFuture;
import io.airlift.log.Logger;
import io.airlift.units.DataSize;
import io.airlift.units.Duration;
import org.weakref.jmx.Flatten;
import org.weakref.jmx.Managed;

import javax.annotation.PostConstruct;
import javax.annotation.PreDestroy;
import javax.inject.Inject;

import java.io.File;
import java.io.IOException;
import java.nio.file.FileAlreadyExistsException;
import java.nio.file.Files;
import java.util.Comparator;
import java.util.Objects;
import java.util.Optional;
import java.util.OptionalLong;
import java.util.Set;
import java.util.UUID;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Future;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.ThreadLocalRandom;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.function.Consumer;

import static com.facebook.presto.raptor.RaptorErrorCode.RAPTOR_RECOVERY_ERROR;
import static com.google.common.base.MoreObjects.toStringHelper;
import static com.google.common.util.concurrent.MoreExecutors.directExecutor;
import static io.airlift.concurrent.Threads.daemonThreadsNamed;
import static io.airlift.units.DataSize.Unit.BYTE;
import static io.airlift.units.DataSize.succinctBytes;
import static io.airlift.units.DataSize.succinctDataSize;
import static io.airlift.units.Duration.nanosSince;
import static java.nio.file.StandardCopyOption.ATOMIC_MOVE;
import static java.util.Objects.requireNonNull;
import static java.util.concurrent.Executors.newCachedThreadPool;
import static java.util.concurrent.Executors.newScheduledThreadPool;
import static java.util.concurrent.TimeUnit.SECONDS;
import static java.util.stream.Collectors.toSet;

public class ShardRecoveryManager
{
    private static final Logger log = Logger.get(ShardRecoveryManager.class);

    private final StorageService storageService;
    private final Optional<BackupStore> backupStore;
    private final String nodeIdentifier;
    private final ShardManager shardManager;
    private final Duration missingShardDiscoveryInterval;

    private final AtomicBoolean started = new AtomicBoolean();
    private final MissingShardsQueue shardQueue;

    private final ScheduledExecutorService missingShardExecutor = newScheduledThreadPool(1, daemonThreadsNamed("missing-shard-discovery"));
    private final ExecutorService executorService = newCachedThreadPool(daemonThreadsNamed("shard-recovery-%s"));
    private final ShardRecoveryStats stats;

    @Inject
    public ShardRecoveryManager(
            StorageService storageService,
            Optional<BackupStore> backupStore,
            NodeManager nodeManager,
            ShardManager shardManager,
            StorageManagerConfig config)
    {
        this(storageService,
                backupStore,
                nodeManager,
                shardManager,
                config.getMissingShardDiscoveryInterval(),
                config.getRecoveryThreads());
    }

    public ShardRecoveryManager(
            StorageService storageService,
            Optional<BackupStore> backupStore,
            NodeManager nodeManager,
            ShardManager shardManager,
            Duration missingShardDiscoveryInterval,
            int recoveryThreads)
    {
        this.storageService = requireNonNull(storageService, "storageService is null");
        this.backupStore = requireNonNull(backupStore, "backupStore is null");
        this.nodeIdentifier = requireNonNull(nodeManager, "nodeManager is null").getCurrentNode().getNodeIdentifier();
        this.shardManager = requireNonNull(shardManager, "shardManager is null");
        this.missingShardDiscoveryInterval = requireNonNull(missingShardDiscoveryInterval, "missingShardDiscoveryInterval is null");
        this.shardQueue = new MissingShardsQueue(new PrioritizedFifoExecutor<>(executorService, recoveryThreads, new MissingShardComparator()));
        this.stats = new ShardRecoveryStats();
    }

    @PostConstruct
    public void start()
    {
        if (!backupStore.isPresent()) {
            return;
        }
        if (started.compareAndSet(false, true)) {
            scheduleRecoverMissingShards();
        }
    }

    @PreDestroy
    public void shutdown()
    {
        executorService.shutdownNow();
        missingShardExecutor.shutdownNow();
    }

    private void scheduleRecoverMissingShards()
    {
        missingShardExecutor.scheduleWithFixedDelay(() -> {
            try {
                // jitter to avoid overloading database
                long interval = missingShardDiscoveryInterval.roundTo(SECONDS);
                SECONDS.sleep(ThreadLocalRandom.current().nextLong(1, interval));
            }
            catch (InterruptedException e) {
                Thread.currentThread().interrupt();
            }
            enqueueMissingShards();
        }, 0, missingShardDiscoveryInterval.toMillis(), TimeUnit.MILLISECONDS);
    }

    @Managed
    public void recoverMissingShards()
    {
        missingShardExecutor.submit(this::enqueueMissingShards);
    }

    private synchronized void enqueueMissingShards()
    {
        try {
            for (ShardMetadata shard : getMissingShards()) {
                stats.incrementBackgroundShardRecovery();
                Futures.addCallback(
                        shardQueue.submit(MissingShard.createBackgroundMissingShard(shard.getShardUuid(), shard.getCompressedSize())),
                        failureCallback(t -> log.warn(t, "Error recovering shard: %s", shard.getShardUuid())));
            }
        }
        catch (Throwable t) {
            log.error(t, "Error creating shard recovery tasks");
        }
    }

    private Set<ShardMetadata> getMissingShards()
    {
        return shardManager.getNodeShards(nodeIdentifier).stream()
                .filter(shard -> shardNeedsRecovery(shard.getShardUuid(), shard.getCompressedSize()))
                .collect(toSet());
    }

    private boolean shardNeedsRecovery(UUID shardUuid, long shardSize)
    {
        File storageFile = storageService.getStorageFile(shardUuid);
        return !storageFile.exists() || (storageFile.length() != shardSize);
    }

    public Future<?> recoverShard(UUID shardUuid)
            throws ExecutionException
    {
        requireNonNull(shardUuid, "shardUuid is null");
        stats.incrementActiveShardRecovery();
        return shardQueue.submit(MissingShard.createActiveMissingShard(shardUuid));
    }

    @VisibleForTesting
    void restoreFromBackup(UUID shardUuid, OptionalLong shardSize)
    {
        File storageFile = storageService.getStorageFile(shardUuid);

        if (!backupStore.get().shardExists(shardUuid)) {
            stats.incrementShardRecoveryBackupNotFound();
            throw new PrestoException(RAPTOR_RECOVERY_ERROR, "No backup file found for shard: " + shardUuid);
        }

        if (storageFile.exists()) {
            if (!shardSize.isPresent() || (storageFile.length() == shardSize.getAsLong())) {
                return;
            }
            log.warn("Local shard file is corrupt. Deleting local file: %s", storageFile);
            storageFile.delete();
        }

        // create a temporary file in the staging directory
        File stagingFile = temporarySuffix(storageService.getStagingFile(shardUuid));
        storageService.createParents(stagingFile);

        // copy to temporary file
        log.info("Copying shard %s from backup...", shardUuid);
        long start = System.nanoTime();

        try {
            backupStore.get().restoreShard(shardUuid, stagingFile);
        }
        catch (PrestoException e) {
            stats.incrementShardRecoveryFailure();
            stagingFile.delete();
            throw e;
        }

        Duration duration = nanosSince(start);
        DataSize size = succinctBytes(stagingFile.length());
        DataSize rate = dataRate(size, duration).convertToMostSuccinctDataSize();
        stats.addShardRecoveryDataRate(rate, size, duration);

        log.info("Copied shard %s from backup in %s (%s at %s/s)", shardUuid, duration, size, rate);

        // move to final location
        storageService.createParents(storageFile);
        try {
            Files.move(stagingFile.toPath(), storageFile.toPath(), ATOMIC_MOVE);
        }
        catch (FileAlreadyExistsException e) {
            // someone else already created it (should not happen, but safe to ignore)
        }
        catch (IOException e) {
            stats.incrementShardRecoveryFailure();
            throw new PrestoException(RAPTOR_RECOVERY_ERROR, "Failed to move shard: " + shardUuid, e);
        }
        finally {
            stagingFile.delete();
        }

        if (!storageFile.exists() || (shardSize.isPresent() && (storageFile.length() != shardSize.getAsLong()))) {
            stats.incrementShardRecoveryFailure();
            log.info("Files do not match after recovery. Deleting local file: " + shardUuid);
            storageFile.delete();
            throw new PrestoException(RAPTOR_RECOVERY_ERROR, "File not recovered correctly: " + shardUuid);
        }

        stats.incrementShardRecoverySuccess();
    }

    @VisibleForTesting
    static class MissingShardComparator
            implements Comparator<MissingShardRunnable>
    {
        @Override
        public int compare(MissingShardRunnable shard1, MissingShardRunnable shard2)
        {
            if (shard1.isActive() == shard2.isActive()) {
                return 0;
            }
            return shard1.isActive() ? -1 : 1;
        }
    }

    interface MissingShardRunnable
            extends Runnable
    {
        boolean isActive();
    }

    private class MissingShardRecovery
            implements MissingShardRunnable
    {
        private final UUID shardUuid;
        private final OptionalLong shardSize;
        private final boolean active;

        public MissingShardRecovery(UUID shardUuid, OptionalLong shardSize, boolean active)
        {
            this.shardUuid = requireNonNull(shardUuid, "shardUuid is null");
            this.shardSize = requireNonNull(shardSize, "shardSize is null");
            this.active = active;
        }

        @Override
        public void run()
        {
            restoreFromBackup(shardUuid, shardSize);
        }

        @Override
        public boolean isActive()
        {
            return active;
        }
    }

    private static final class MissingShard
    {
        private final UUID shardUuid;
        private final OptionalLong shardSize;
        private final boolean active;

        private MissingShard(UUID shardUuid, OptionalLong shardSize, boolean active)
        {
            this.shardUuid = requireNonNull(shardUuid, "shardUuid is null");
            this.shardSize = requireNonNull(shardSize, "shardSize is null");
            this.active = active;
        }

        public static MissingShard createBackgroundMissingShard(UUID shardUuid, long shardSize)
        {
            return new MissingShard(shardUuid, OptionalLong.of(shardSize), false);
        }

        public static MissingShard createActiveMissingShard(UUID shardUuid)
        {
            return new MissingShard(shardUuid, OptionalLong.empty(), true);
        }

        public UUID getShardUuid()
        {
            return shardUuid;
        }

        public OptionalLong getShardSize()
        {
            return shardSize;
        }

        public boolean isActive()
        {
            return active;
        }

        @Override
        public boolean equals(Object o)
        {
            if (this == o) {
                return true;
            }
            if (o == null || getClass() != o.getClass()) {
                return false;
            }

            MissingShard other = (MissingShard) o;
            return Objects.equals(this.active, other.active) &&
                    Objects.equals(this.shardUuid, other.shardUuid);
        }

        @Override
        public int hashCode()
        {
            return Objects.hash(shardUuid, active);
        }

        @Override
        public String toString()
        {
            return toStringHelper(this)
                    .add("shardUuid", shardUuid)
                    .add("active", active)
                    .toString();
        }
    }

    private class MissingShardsQueue
    {
        private final LoadingCache<MissingShard, ListenableFuture<?>> queuedMissingShards;

        public MissingShardsQueue(PrioritizedFifoExecutor<MissingShardRunnable> shardRecoveryExecutor)
        {
            requireNonNull(shardRecoveryExecutor, "shardRecoveryExecutor is null");
            this.queuedMissingShards = CacheBuilder.newBuilder().build(new CacheLoader<MissingShard, ListenableFuture<?>>()
            {
                @Override
                public ListenableFuture<?> load(MissingShard missingShard)
                {
                    MissingShardRecovery task = new MissingShardRecovery(
                            missingShard.getShardUuid(),
                            missingShard.getShardSize(),
                            missingShard.isActive());
                    ListenableFuture<?> future = shardRecoveryExecutor.submit(task);
                    future.addListener(() -> queuedMissingShards.invalidate(missingShard), directExecutor());
                    return future;
                }
            });
        }

        public ListenableFuture<?> submit(MissingShard shard)
                throws ExecutionException
        {
            return queuedMissingShards.get(shard);
        }
    }

    static DataSize dataRate(DataSize size, Duration duration)
    {
        double rate = size.toBytes() / duration.getValue(SECONDS);
        if (Double.isNaN(rate) || Double.isInfinite(rate)) {
            rate = 0;
        }
        return succinctDataSize(rate, BYTE);
    }

    private static File temporarySuffix(File file)
    {
        return new File(file.getPath() + ".tmp-" + UUID.randomUUID());
    }

    @Managed
    @Flatten
    public ShardRecoveryStats getStats()
    {
        return stats;
    }

    private static <T> FutureCallback<T> failureCallback(Consumer<Throwable> callback)
    {
        return new FutureCallback<T>()
        {
            @Override
            public void onSuccess(T result) {}

            @Override
            public void onFailure(Throwable throwable)
            {
                callback.accept(throwable);
            }
        };
    }
}


text:
            {{ trans('auth::dashboard.titles.permissions-total') }}

        {{ $permissionsCount }}


text:
package de.chrgroth.jsonstore.json.flexjson.custom;

import java.lang.reflect.Type;
import java.time.format.DateTimeFormatter;
import java.util.Date;

import flexjson.ObjectBinder;

/**
 * Custom type handler processing {@link Date} instances with configured pattern.
 *
 * @author Christian Groth
 */
public class DateTypeHandler extends AbstractFlexjsonTypeHandler {

    private flexjson.transformer.DateTransformer delegate;

    /**
     * Creates a new instance using the given pattern. Take a look at {@link DateTimeFormatter} for concrete syntax.
     *
     * @param dateTimePattern
     *            date time pattern
     */
    public DateTypeHandler(String dateTimePattern) {
        delegate = new flexjson.transformer.DateTransformer(dateTimePattern);
    }

    @Override
    public void transform(Object object) {
        delegate.transform(object);
    }

    @Override
    public Object instantiate(ObjectBinder context, Object value, Type targetType, @SuppressWarnings("rawtypes") Class targetClass) {
        return delegate.instantiate(context, value, targetType, targetClass);
    }
}


text:
var successes = {
  applySortEvents : function() {
    $('#tblToBeSort').each(function() {
        var $table = $(this);                       //将table存储为一个jquery对象

        $('th', $table).each(function(column) {
            var findSortKey;
            if ($(this).is('.sort-alpha')) {                //按字母排序
                findSortKey = function($cell) {
                    return $cell.find('sort-key').text().toUpperCase() + '' + $cell.text().toUpperCase();
                };
            } else if ($(this).is('.sort-numeric')) {       //按数字排序
                findSortKey = function($cell) {
                    var key = parseFloat($cell.text().replace(/^[^\d.]*/, ''));
                    return isNaN(key) ? 0 : key;
                };
            } else if ($(this).is('.sort-date')) {          //按日期排序
                findSortKey = function($cell) {
                    return Date.parse('1 ' + $cell.text());
                };
            } else if ($(this).find('select').length > 0) {
                // yuqing 2014/4/15: add success type filter here
                var filterSel = $(this).find('select');
                filterSel.change(function() {
                  $("#tblToBeSort tbody tr").each(function() {
                      var typeText = $(this).children('td').eq(column).text();
                      var keywords = filterSel.find("option:selected").val().split(' ');
                      var i = 0;
                      for (; i < keywords.length; ++i) {
                        if (typeText.indexOf(keywords[i]) >= 0) {
                          break;
                        }
                      }
                      if (i < keywords.length) {
                        $(this).show();
                      } else {
                        $(this).hide();
                      }
                  });
                });
            }

            if (findSortKey) {
                $(this).addClass('clickable').hover(function() { $(this).addClass('hover'); }, function() { $(this).removeClass('hover'); }).click(function() {
                    //反向排序状态声明
                    var newDirection = 1;
                    if ($(this).is('.sorted-asc')) {
                        newDirection = -1;
                    }
                    var rows = $table.find('tbody>tr').get(); //将数据行转换为数组
                    $.each(rows, function(index, row) {
                        row.sortKey = findSortKey($(row).children('td').eq(column));
                    });
                    rows.sort(function(a, b) {
                        if (a.sortKey  b.sortKey) return newDirection;
                        return 0;
                    });
                    $.each(rows, function(index, row) {
                        $table.children('tbody').append(row);
                        row.sortKey = null;
                    });

                    $table.find('th').removeClass('sorted-asc').removeClass('sorted-desc');
                    var $sortHead = $table.find('th').filter(':nth-child(' + (column + 1) + ')');
                    //实现反向排序
                    if (newDirection == 1) {
                        $sortHead.addClass('sorted-asc');
                    } else {
                        $sortHead.addClass('sorted-desc');
                    }

                    //移除已排序的列的样式,添加样式到当前列
                    $table.find('td').removeClass('sorted').filter(':nth-child(' + (column + 1) + ')').addClass('sorted');
                    $table.trigger('repaginate');
                });
            }
        });
    });
  }
};

$(function() {
  // load left navi items and content divs
  // NOTE: this "ready" callback must be assigned after the "ready" callback in leftnavi.js
  $.ajax({
    type: "GET",
    dataType: "json",
    url: "/backend/successes-leftNavi",
    success: function(result) {
      if (result.length === 5) {
        $("#bodycontainer div.leftpanel div.leftnavi ul li a:lt(5)").text(function(i){
          return result[i].short_title;
        });
        $("#bodycontainer div.content div.item p.title:lt(5)").text(function(i){
          return result[i].title;
        });
        $("#bodycontainer div.content div.item:lt(5)").append(function(i){
          return $("
/backend/successes-detailById?id=" + result[i].id + "
");
        });
        // reload the current item div, the leftnavi.js will delay-load the content of it.
        leftnavi.showcontent(
            leftnavi.contentmap.get(document.location.hash)
            );
      }
    }
  });
});


text:
require 'spec_helper'
require 'whois/parsers/whois.nic.design.rb'

describe Whois::Parsers::WhoisNicDesign, "status_registered.expected" do

  subject do
    file = fixture("responses", "whois.nic.design/design/status_registered.txt")
    part = Whois::Record::Part.new(body: File.read(file))
    described_class.new(part)
  end

  describe "#disclaimer" do
    it do
      expect(subject.disclaimer).to eq("Access to the whois service is rate limited. For more information, please see https://registrar-console.centralnic.com/pub/whois_guidance.")
    end
  end
  describe "#domain" do
    it do
      expect(subject.domain).to eq("toplevel.design")
    end
  end
  describe "#domain_id" do
    it do
      expect(subject.domain_id).to eq("D7069819-CNIC")
    end
  end
  describe "#status" do
    it do
      expect(subject.status).to eq(["ok"])
    end
  end
  describe "#available?" do
    it do
      expect(subject.available?).to eq(false)
    end
  end
  describe "#registered?" do
    it do
      expect(subject.registered?).to eq(true)
    end
  end
  describe "#created_on" do
    it do
      expect(subject.created_on).to be_a(Time)
      expect(subject.created_on).to eq(Time.parse("2015-02-27 16:08:32 UTC"))
    end
  end
  describe "#updated_on" do
    it do
      expect(subject.updated_on).to be_a(Time)
      expect(subject.updated_on).to eq(Time.parse("2015-04-21 17:48:34 UTC"))
    end
  end
  describe "#expires_on" do
    it do
      expect(subject.expires_on).to be_a(Time)
      expect(subject.expires_on).to eq(Time.parse("2016-02-27 23:59:59 UTC"))
    end
  end
  describe "#registrar" do
    it do
      expect(subject.registrar).to be_a(Whois::Parser::Registrar)
      expect(subject.registrar.id).to eq("9999")
      expect(subject.registrar.name).to eq("Top Level Design, LLC")
      expect(subject.registrar.organization).to eq(nil)
      expect(subject.registrar.url).to eq(nil)
    end
  end
  describe "#registrant_contacts" do
    it do
      expect(subject.registrant_contacts).to be_a(Array)
      expect(subject.registrant_contacts.size).to eq(1)
      expect(subject.registrant_contacts[0]).to be_a(Whois::Parser::Contact)
      expect(subject.registrant_contacts[0].type).to eq(Whois::Parser::Contact::TYPE_REGISTRANT)
      expect(subject.registrant_contacts[0].id).to eq("H4596017")
      expect(subject.registrant_contacts[0].name).to eq("Domain Administrator")
      expect(subject.registrant_contacts[0].organization).to eq("Top Level Design, LLC")
      expect(subject.registrant_contacts[0].address).to eq("742 Ocean Club Place")
      expect(subject.registrant_contacts[0].city).to eq("Fernandina Beach")
      expect(subject.registrant_contacts[0].zip).to eq("32034")
      expect(subject.registrant_contacts[0].state).to eq("Florida")
      expect(subject.registrant_contacts[0].country).to eq(nil)
      expect(subject.registrant_contacts[0].country_code).to eq("US")
      expect(subject.registrant_contacts[0].phone).to eq("+1.5038888808")
      expect(subject.registrant_contacts[0].fax).to eq("+1.6788841468")
      expect(subject.registrant_contacts[0].email).to eq("ray@tldesign.co")
    end
  end
  describe "#admin_contacts" do
    it do
      expect(subject.admin_contacts).to be_a(Array)
      expect(subject.admin_contacts.size).to eq(1)
      expect(subject.admin_contacts[0]).to be_a(Whois::Parser::Contact)
      expect(subject.admin_contacts[0].type).to eq(Whois::Parser::Contact::TYPE_ADMINISTRATIVE)
      expect(subject.admin_contacts[0].id).to eq("H4596017")
      expect(subject.admin_contacts[0].name).to eq("Domain Administrator")
      expect(subject.admin_contacts[0].organization).to eq("Top Level Design, LLC")
      expect(subject.admin_contacts[0].address).to eq("742 Ocean Club Place")
      expect(subject.admin_contacts[0].city).to eq("Fernandina Beach")
      expect(subject.admin_contacts[0].zip).to eq("32034")
      expect(subject.admin_contacts[0].state).to eq("Florida")
      expect(subject.admin_contacts[0].country).to eq(nil)
      expect(subject.admin_contacts[0].country_code).to eq("US")
      expect(subject.admin_contacts[0].phone).to eq("+1.5038888808")
      expect(subject.admin_contacts[0].fax).to eq("+1.6788841468")
      expect(subject.admin_contacts[0].email).to eq("ray@tldesign.co")
    end
  end
  describe "#technical_contacts" do
    it do
      expect(subject.technical_contacts).to be_a(Array)
      expect(subject.technical_contacts.size).to eq(1)
      expect(subject.technical_contacts[0]).to be_a(Whois::Parser::Contact)
      expect(subject.technical_contacts[0].type).to eq(Whois::Parser::Contact::TYPE_TECHNICAL)
      expect(subject.technical_contacts[0].id).to eq("H4596017")
      expect(subject.technical_contacts[0].name).to eq("Domain Administrator")
      expect(subject.technical_contacts[0].organization).to eq("Top Level Design, LLC")
      expect(subject.technical_contacts[0].address).to eq("742 Ocean Club Place")
      expect(subject.technical_contacts[0].city).to eq("Fernandina Beach")
      expect(subject.technical_contacts[0].zip).to eq("32034")
      expect(subject.technical_contacts[0].state).to eq("Florida")
      expect(subject.technical_contacts[0].country).to eq(nil)
      expect(subject.technical_contacts[0].country_code).to eq("US")
      expect(subject.technical_contacts[0].phone).to eq("+1.5038888808")
      expect(subject.technical_contacts[0].fax).to eq("+1.6788841468")
      expect(subject.technical_contacts[0].email).to eq("ray@tldesign.co")
    end
  end
  describe "#nameservers" do
    it do
      expect(subject.nameservers).to be_a(Array)
      expect(subject.nameservers.size).to eq(4)
      expect(subject.nameservers[0]).to be_a(Whois::Parser::Nameserver)
      expect(subject.nameservers[0].name).to eq("ns-170.awsdns-21.com")
      expect(subject.nameservers[0].ipv4).to eq(nil)
      expect(subject.nameservers[0].ipv6).to eq(nil)
      expect(subject.nameservers[1]).to be_a(Whois::Parser::Nameserver)
      expect(subject.nameservers[1].name).to eq("ns-904.awsdns-49.net")
      expect(subject.nameservers[1].ipv4).to eq(nil)
      expect(subject.nameservers[1].ipv6).to eq(nil)
      expect(subject.nameservers[2]).to be_a(Whois::Parser::Nameserver)
      expect(subject.nameservers[2].name).to eq("ns-1067.awsdns-05.org")
      expect(subject.nameservers[2].ipv4).to eq(nil)
      expect(subject.nameservers[2].ipv6).to eq(nil)
      expect(subject.nameservers[3]).to be_a(Whois::Parser::Nameserver)
      expect(subject.nameservers[3].name).to eq("ns-1873.awsdns-42.co.uk")
      expect(subject.nameservers[3].ipv4).to eq(nil)
      expect(subject.nameservers[3].ipv6).to eq(nil)
    end
  end
end


text:
static UniColumnType columnTypeOfProperty(Class cls,UniProperty* property){
    switch (property.typeEncoding) {
        case UniTypeEncodingBool:
        case UniTypeEncodingInt8:
        case UniTypeEncodingUInt8:
        case UniTypeEncodingInt16:
        case UniTypeEncodingUInt16:
        case UniTypeEncodingInt32:
        case UniTypeEncodingUInt32:
        case UniTypeEncodingInt64:
        case UniTypeEncodingUInt64:
            if([cls respondsToSelector:@selector(uni_columnType:)]){
                NSCAssert(![cls uni_columnType:property.name], @"property can be process automatically,you should not implement uni_columnType: for property: %@",property.name);
            }
            if([cls respondsToSelector:@selector(uni_dbTransformer:)]){
                NSCAssert(![cls uni_dbTransformer:property.name], @"property can be process automatically,you should not implement uni_dbTransformer: for property: %@",property.name);
            }
            return UniColumnTypeInteger;
        case UniTypeEncodingFloat:
        case UniTypeEncodingDouble:
        case UniTypeEncodingLongDouble:
            if([cls respondsToSelector:@selector(uni_columnType:)]){
                NSCAssert(![cls uni_columnType:property.name], @"property can be process automatically,you should not implement uni_columnType: for property: %@",property.name);
            }
            if([cls respondsToSelector:@selector(uni_dbTransformer:)]){
                NSCAssert(![cls uni_dbTransformer:property.name], @"property can be process automatically,you should not implement uni_dbTransformer: for property: %@",property.name);
            }
            return UniColumnTypeReal;
        case UniTypeEncodingClass:
        case UniTypeEncodingSEL:
        case UniTypeEncodingNSRange:
        case UniTypeEncodingCATansform3D:
        case UniTypeEncodingPoint:
        case UniTypeEncodingSize:
        case UniTypeEncodingRect:
        case UniTypeEncodingEdgeInsets:
        case UniTypeEncodingCGVector:
        case UniTypeEncodingCGAffineTransform:
        case UniTypeEncodingUIOffset:
        case UniTypeEncodingNSDirectionalEdgeInsets:
        case UniTypeEncodingNSString:
        case UniTypeEncodingNSMutableString:
        case UniTypeEncodingNSURL:
        case UniTypeEncodingNSNumber:
        case UniTypeEncodingNSDecimalNumber:
            if([cls respondsToSelector:@selector(uni_columnType:)]){
                NSCAssert(![cls uni_columnType:property.name], @"property can be process automatically,you should not implement uni_columnType: for property: %@",property.name);
            }
            if([cls respondsToSelector:@selector(uni_dbTransformer:)]){
                NSCAssert(![cls uni_dbTransformer:property.name], @"property can be process automatically,you should not implement uni_dbTransformer: for property: %@",property.name);
            }
            return UniColumnTypeText;
        case UniTypeEncodingNSDate:
            if([cls respondsToSelector:@selector(uni_columnType:)]){
                NSCAssert(![cls uni_columnType:property.name], @"property can be process automatically,you should not implement uni_columnType: for property: %@",property.name);
            }
            if([cls respondsToSelector:@selector(uni_dbTransformer:)]){
                NSCAssert(![cls uni_dbTransformer:property.name], @"property can be process automatically,you should not implement uni_dbTransformer: for property: %@",property.name);
            }
            return UniColumnTypeReal;
        case UniTypeEncodingNSData:
        case UniTypeEncodingNSMutableData:
            if([cls respondsToSelector:@selector(uni_columnType:)]){
                NSCAssert(![cls uni_columnType:property.name], @"property can be process automatically,you should not implement uni_columnType: for property: %@",property.name);
            }
            if([cls respondsToSelector:@selector(uni_dbTransformer:)]){
                NSCAssert(![cls uni_dbTransformer:property.name], @"property can be process automatically,you should not implement uni_dbTransformer: for property: %@",property.name);
            }
            return UniColumnTypeBlob;
        case UniTypeEncodingNSObject:{
            UniClass *clz=[[UniClass alloc]initWithClass:property.cls];
            if (clz.isConformingToUniDB||clz.isConformingToUniDB){
                return columnTypeOfProperty(clz.cls, clz.primaryProperty);
            }else{
                property.dbTransformer = [cls uni_dbTransformer:property.name];
                return [cls uni_columnType:property.name];
            }
        }
        default:
            property.dbTransformer = [cls uni_dbTransformer:property.name];
            return [cls uni_columnType:property.name];
    }
}
static __inline__ __attribute__((always_inline)) NSString * uni_encodingDesc(UniTypeEncoding typeEncoding,UniPropertyEncoding propertyEncoding,UniMethodEncoding methodEncoding){
    NSMutableString * type = [NSMutableString string];
    switch (typeEncoding) {
        case UniTypeEncodingBool: [type appendString:@"bool"]; break;
        case UniTypeEncodingInt8: [type appendString:@"int8"]; break;
        case UniTypeEncodingUInt8: [type appendString:@"uint8"]; break;
        case UniTypeEncodingInt16: [type appendString:@"int16"]; break;
        case UniTypeEncodingUInt16: [type appendString:@"uint16"]; break;
        case UniTypeEncodingInt32: [type appendString:@"int32"]; break;
        case UniTypeEncodingUInt32: [type appendString:@"uint32"]; break;
        case UniTypeEncodingInt64: [type appendString:@"int64"]; break;
        case UniTypeEncodingUInt64: [type appendString:@"uint64"]; break;
        case UniTypeEncodingFloat: [type appendString:@"float"]; break;
        case UniTypeEncodingDouble: [type appendString:@"double"]; break;
        case UniTypeEncodingLongDouble: [type appendString:@"longDouble"]; break;
        case UniTypeEncodingClass: [type appendString:@"class"]; break;
        case UniTypeEncodingSEL: [type appendString:@"sel"]; break;
        case UniTypeEncodingNSRange: [type appendString:@"range"]; break;
        case UniTypeEncodingCATansform3D: [type appendString:@"tansform3D"]; break;
        case UniTypeEncodingPoint: [type appendString:@"point"]; break;
        case UniTypeEncodingSize: [type appendString:@"size"]; break;
        case UniTypeEncodingRect: [type appendString:@"rect"]; break;
        case UniTypeEncodingEdgeInsets: [type appendString:@"edgeInsets"]; break;
        case UniTypeEncodingCGVector: [type appendString:@"vector"]; break;
        case UniTypeEncodingCGAffineTransform: [type appendString:@"affineTransform"]; break;
        case UniTypeEncodingUIOffset: [type appendString:@"UIOffset"]; break;
        case UniTypeEncodingNSDirectionalEdgeInsets: [type appendString:@"directionalEdgeInsets"]; break;
        case UniTypeEncodingBlock: [type appendString:@"block"]; break;
        case UniTypeEncodingNSString: [type appendString:@"string"]; break;
        case UniTypeEncodingNSMutableString: [type appendString:@"mutable string"]; break;
        case UniTypeEncodingNSNumber: [type appendString:@"number"]; break;
        case UniTypeEncodingNSDecimalNumber: [type appendString:@"decimal number"]; break;
        case UniTypeEncodingNSDate: [type appendString:@"date"]; break;
        case UniTypeEncodingNSData: [type appendString:@"data"]; break;
        case UniTypeEncodingNSMutableData:[type appendString:@"mutable data"]; break;
        case UniTypeEncodingNSOrderedSet:[type appendString:@"ordered set"];break;
        case UniTypeEncodingNSArray: [type appendString:@"array"]; break;
        case UniTypeEncodingNSMutableArray: [type appendString:@"mutable array"]; break;
        case UniTypeEncodingNSSet: [type appendString:@"set"]; break;
        case UniTypeEncodingNSMutableSet: [type appendString:@"mutable set"]; break;
        case UniTypeEncodingNSDictionary: [type appendString:@"dictionary"]; break;
        case UniTypeEncodingNSMutableDictionary: [type appendString:@"mutable dictionary"]; break;
        case UniTypeEncodingPointer: [type appendString:@"pointer"]; break;
        case UniTypeEncodingStruct: [type appendString:@"struct"]; break;
        case UniTypeEncodingUnion: [type appendString:@"union"]; break;
        case UniTypeEncodingCString: [type appendString:@"cstring"]; break;
        case UniTypeEncodingCArray: [type appendString:@"carray"]; break;
        case UniTypeEncodingNSObject: [type appendString:@"object"]; break;
        default: [type appendString:@"unknow"]; break;
    }
    if(methodEncoding&UniMethodEncodingConst) [type appendString:@" const"];
    if(methodEncoding&UniMethodEncodingIn) [type appendString:@" in"];
    if(methodEncoding&UniMethodEncodingInout) [type appendString:@" inout"];
    if(methodEncoding&UniMethodEncodingBycopy) [type appendString:@" byCopy"];
    if(methodEncoding&UniMethodEncodingByref) [type appendString:@" byref"];
    if(methodEncoding&UniMethodEncodingOneway) [type appendString:@" oneway"];
    if(propertyEncoding&UniPropertyEncodingReadonly) [type appendString:@" readonly"];
    if(propertyEncoding&UniPropertyEncodingCopy) [type appendString:@" copy"];
    if(propertyEncoding&UniPropertyEncodingRetain) [type appendString:@" retain"];
    if(propertyEncoding&UniPropertyEncodingNonatomic) [type appendString:@" nonatomic"];
    if(propertyEncoding&UniPropertyEncodingWeak) [type appendString:@" weak"];
    if(propertyEncoding&UniPropertyEncodingCustomGetter) [type appendString:@" customGetter"];
    if(propertyEncoding&UniPropertyEncodingCustomSetter) [type appendString:@" customSetter"];
    if(propertyEncoding&UniPropertyEncodingDynamic) [type appendString:@" dynamic"];
    return type;
}

static __inline__ __attribute__((always_inline)) NSString * uni_columnDesc(UniColumnType columnType){
    static NSArray * arr=nil;
    static dispatch_once_t onceToken;
    dispatch_once(&onceToken, ^{arr=@[@"automatically",@"integer",@"real",@"text",@"blob"];});
    return arr[columnType];
}

static __inline__ __attribute__((always_inline)) NSDictionary * uni_columns_of_table(UniDB *db, NSString *table){
    NSArray *dics = [db executeQuery:@"SELECT * FROM sqlite_master WHERE tbl_name=? AND type='table'" arguments:@[table] error:nil];
    NSDictionary *dic=dics.lastObject;
    NSString *sql=dic[@"sql"];
    if (!sql) return nil;
    NSScanner *scanner=[NSScanner scannerWithString:sql];
    if(![scanner scanUpToString:@"(" intoString:nil]) return nil;
    NSMutableDictionary *dict=[NSMutableDictionary dictionary];
    while (![scanner isAtEnd]) {
        NSString *column=nil;
        NSString *type=nil;
        if(![scanner scanUpToString:@"`" intoString:nil]) break;
        if (scanner.scanLocation>=scanner.string.length) break;
        scanner.scanLocation++;
        if(![scanner scanUpToString:@"`" intoString:&column]) break;
        if (scanner.scanLocation>=scanner.string.length) break;
        if(![scanner scanUpToString:@" " intoString:nil]) break;
        scanner.scanLocation++;
        if(![scanner scanUpToCharactersFromSet:[NSCharacterSet characterSetWithCharactersInString:@" ,)"] intoString:&type]) break;
        dict[column]=type;
    }
    return dict;
}

static __inline__ __attribute__((always_inline)) NSDictionary * uni_indexes_of_table(UniDB *db, NSString *table){
    NSArray *dics = [db executeQuery:@"SELECT * FROM sqlite_master WHERE tbl_name=? AND type='index'" arguments:@[table] error:nil];
    NSMutableDictionary *dict=[NSMutableDictionary dictionary];
    for (NSDictionary *dic in dics){
        NSString *index=dic[@"name"];
        dict[index]=index;
    }
    return dict;
}

@interface UniClass ()

@end

@interface UniProperty ()

- (instancetype)initWithProperty:(objc_property_t)property;

@end

@implementation UniClass

+ (instancetype)classWithClass:(Class)cls{
    if (!cls) return nil;
    static CFMutableDictionaryRef classCache;
    static dispatch_once_t onceToken;
    static dispatch_semaphore_t semaphore;
    dispatch_once(&onceToken, ^{
        classCache = CFDictionaryCreateMutable(CFAllocatorGetDefault(), 0, &kCFTypeDictionaryKeyCallBacks, &kCFTypeDictionaryValueCallBacks);
        semaphore = dispatch_semaphore_create(1);
    });
    dispatch_semaphore_wait(semaphore, DISPATCH_TIME_FOREVER);
    UniClass *clz = CFDictionaryGetValue(classCache, (__bridge const void *)(cls));
    dispatch_semaphore_signal(semaphore);
    if (!clz) {
        clz = [[UniClass alloc] initWithClass:cls];
        if (!clz) return nil;
        dispatch_semaphore_wait(semaphore, DISPATCH_TIME_FOREVER);
        if (!CFDictionaryGetValue(classCache, (__bridge const void *)(cls))) {
            [clz prepare];
            CFDictionarySetValue(classCache, (__bridge const void *)(cls), (__bridge const void *)(clz));
        }
        dispatch_semaphore_signal(semaphore);
    }
    return clz;
}

- (instancetype)initWithClass:(Class)cls{
    self=[self init];
    if (!self) return nil;
    self.cls=cls;
    self.name=NSStringFromClass(cls);
    NSMutableArray *propertyArr=[NSMutableArray array];
    NSMutableDictionary *propertyDic=[NSMutableDictionary dictionary];
    [self enumeratePropertiesUsingBlock:^(objc_property_t p) {
        UniProperty *property=[[UniProperty alloc]initWithProperty:p];
        if (!property.setter) return;
        [propertyArr addObject:property];
        propertyDic[property.name]=property;
    }];
    self.propertyArr=propertyArr;
    self.propertyDic=propertyDic;
    if ([cls conformsToProtocol:@protocol(UniJSON)]) self.isConformingToUniJSON=YES;
    if ([cls conformsToProtocol:@protocol(UniMM)]) self.isConformingToUniMM=YES;
    if ([cls conformsToProtocol:@protocol(UniDB)]) self.isConformingToUniDB=YES;
    if (self.isConformingToUniMM||self.isConformingToUniDB) self.primaryProperty=self.propertyDic[[self.cls uni_primaryKey]];
    return self;
}

- (void)prepare{
    if (self.isConformingToUniJSON) {
        NSMutableArray *arr=[NSMutableArray array];
        [[self.cls uni_keyPaths] enumerateKeysAndObjectsUsingBlock:^(id  _Nonnull key, id  _Nonnull obj, BOOL * _Nonnull stop) {
            NSMutableArray *jsonKeyPathArr=[NSMutableArray array];
            if ([obj isKindOfClass:NSString.class]) [jsonKeyPathArr addObject:[obj componentsSeparatedByString:@"."]];
            else if([obj isKindOfClass:NSArray.class]) [obj enumerateObjectsUsingBlock:^(id  _Nonnull obj1, NSUInteger idx, BOOL * _Nonnull stop) {
                [jsonKeyPathArr addObject:[obj1 componentsSeparatedByString:@"."]];
            }];
            UniProperty *property=self.propertyDic[key];
            property.jsonKeyPathArr=jsonKeyPathArr;
            [arr addObject:property];
            if ([self.cls respondsToSelector:@selector(uni_jsonTransformer:)]) property.jsonTransformer=[self.cls uni_jsonTransformer:key];
        }];
        self.jsonPropertyArr=arr;
    }
    if (self.isConformingToUniMM)  self.mm=[NSMapTable strongToWeakObjectsMapTable];
    if (self.isConformingToUniDB) {
        NSMutableArray *arr=[NSMutableArray array];
        [[self.cls uni_columns] enumerateObjectsUsingBlock:^(id  _Nonnull obj, NSUInteger idx, BOOL * _Nonnull stop) {
            UniProperty *property=self.propertyDic[obj];
            property.columnType=columnTypeOfProperty(self.cls, property);
            [arr addObject:property];
        }];
        self.dbPropertyArr=arr;
        self.dbSelectSql = [NSString stringWithFormat:@"SELECT * FROM `%@` WHERE `%@`=?;", self.name, self.primaryProperty.name];
        NSMutableString *sql = [NSMutableString stringWithFormat:@"UPDATE `%@` SET ", self.name];
        NSMutableString *sql1 = [NSMutableString stringWithFormat:@"INSERT INTO `%@` (", self.name];
        NSMutableString *sql2 = [NSMutableString stringWithFormat:@" VALUES ("];
        for (UniProperty *property in self.dbPropertyArr){
            [sql appendFormat:@"`%@`=?,", property.name];
            [sql1 appendFormat:@"`%@`,", property.name];
            [sql2 appendFormat:@"?,"];
        }
        [sql appendFormat:@"`%@`=?,", @"uni_update_at"];
        [sql deleteCharactersInRange:NSMakeRange(sql.length - 1, 1)];
        [sql appendFormat:@" WHERE `%@`=?;", self.primaryProperty.name];
        self.dbUpdateSql = sql;
        [sql1 appendFormat:@"`%@`,", @"uni_update_at"];
        [sql2 appendFormat:@"?,"];
        [sql1 deleteCharactersInRange:NSMakeRange(sql1.length - 1, 1)];
        [sql1 appendFormat:@")"];
        [sql2 deleteCharactersInRange:NSMakeRange(sql2.length - 1, 1)];
        [sql2 appendFormat:@")"];
        self.dbInsertSql = [NSString stringWithFormat:@"%@%@;", sql1, sql2];
        self.dbDeleteSql = [NSString stringWithFormat:@"DELETE FROM %@ WHERE `%@`=?;",self.name,self.primaryProperty.name];
        self.db=[[UniDB alloc]init];
        [self openAndInit:[NSSearchPathForDirectoriesInDomains(NSLibraryDirectory, NSUserDomainMask, YES).lastObject stringByAppendingPathComponent:[NSString stringWithFormat:@"Unicorn/%@.sqlite3",self.name]] error:nil];
    }
    if (self.primaryProperty) {
        NSAssert(!self.primaryProperty.jsonTransformer,@"primary property do not support transformer");
        NSAssert(!self.primaryProperty.dbTransformer,@"primary property do not support transformer");
        NSAssert({
               BOOL valid=NO;
            switch (self.primaryProperty.typeEncoding) {
                case UniTypeEncodingBool:
                case UniTypeEncodingInt8:
                case UniTypeEncodingUInt8:
                case UniTypeEncodingInt16:
                case UniTypeEncodingUInt16:
                case UniTypeEncodingInt32:
                case UniTypeEncodingUInt32:
                case UniTypeEncodingInt64:
                case UniTypeEncodingUInt64:
                case UniTypeEncodingFloat:
                case UniTypeEncodingDouble:
                case UniTypeEncodingLongDouble:
                case UniTypeEncodingClass:
                case UniTypeEncodingSEL:
                case UniTypeEncodingNSString:
                case UniTypeEncodingNSMutableString:
                case UniTypeEncodingNSURL:
                    valid=YES;
                default:
                    break;
            }
               valid;
            }, @"primary property type must be above");
    }
}

- (void)enumeratePropertiesUsingBlock:(void (^)(objc_property_t property))block {
    Class cls = self.cls;
    while (YES) {
        if (cls == NSObject.class) break;
        unsigned int count = 0;
        objc_property_t *properties = class_copyPropertyList(cls, &count);
        if (properties == NULL) { cls = cls.superclass; continue; }
        for (unsigned i = 0; i < count; i++) {
            objc_property_t property = properties[i];
            block(property);
        }
        free(properties);
        cls = cls.superclass;
    }
}

- (BOOL)openAndInit:(NSString*)file error:(NSError**)error{
    NSString *path=[file stringByDeletingLastPathComponent];
    [[NSFileManager defaultManager] createDirectoryAtPath:path withIntermediateDirectories:YES attributes:nil error:nil];
    if(![self.db open:file error:error]){ NSAssert(0,@"db error %@",*error); return NO; }
    NSString *primaryColumnType = uni_columnDesc(self.primaryProperty.columnType);
    NSMutableDictionary *oldColumns = [uni_columns_of_table(self.db, self.name) mutableCopy];
    NSMutableDictionary *oldIndexes = [uni_indexes_of_table(self.db, self.name) mutableCopy];
    NSMutableArray *addPropertyArr=[NSMutableArray array];
    NSMutableArray *addIndexArr=[NSMutableArray array];
    __block NSString *sql = nil;
    if (oldColumns.count>0&&(!oldColumns[self.primaryProperty.name]||![oldColumns[self.primaryProperty.name] isEqualToString:primaryColumnType])) {
        NSAssert(0, @"dangerous operation,if the primary key changed,table will be drop and recreate");
        [self.db executeUpdate:[NSString stringWithFormat:@"DROP TABLE `%@`", self.name] arguments:nil error:nil];
    }
    [self.db executeUpdate:[NSString stringWithFormat:@"CREATE TABLE IF NOT EXISTS `%@` (`%@` %@ NOT NULL PRIMARY KEY)", self.name, self.primaryProperty.name, primaryColumnType] arguments:nil error:nil];
    if (!oldColumns) oldColumns=[NSMutableDictionary dictionary];
    oldColumns[self.primaryProperty.name]=primaryColumnType;
    NSString *add=@"ALTER TABLE `%@` ADD COLUMN `%@` %@";
    for (UniProperty *property in self.dbPropertyArr){
        NSString *type=oldColumns[property.name];
        if (!type) [addPropertyArr addObject:property];
        else if (![type isEqualToString:uni_columnDesc(property.columnType)]){
                NSAssert(0, @"column: %@,old column type: %@,new column type: %@",property.name,type,uni_columnDesc(property.columnType));
        }
    }
    //add necessary columns
    for (UniProperty *property in addPropertyArr){
        sql=[NSString stringWithFormat:add,self.name,property.name,uni_columnDesc(property.columnType)];
        [self.db executeUpdate:sql arguments:nil error:nil];
    }
    if (!oldColumns[@"uni_update_at"]) {
        sql=[NSString stringWithFormat:add,self.name,@"uni_update_at",uni_columnDesc(UniColumnTypeReal)];
        [self.db executeUpdate:sql arguments:nil error:nil];
    }
    NSMutableArray *indexes=[NSMutableArray array];
    if ([self.cls respondsToSelector:@selector(uni_indexes)]) [indexes addObjectsFromArray:[self.cls uni_indexes]];
    [indexes addObject:@"uni_update_at"];
    for (NSString *idx in indexes){
        if (!oldIndexes[idx]){
            [addIndexArr addObject:@{@"indexname":idx,@"index":[[idx componentsSeparatedByString:@","] componentsJoinedByString:@"`,`"]}];
        }
        else oldIndexes[idx]=nil;
    }
    //add necessary indexes
    for (NSDictionary * idx in addIndexArr) [self.db executeUpdate:[NSString stringWithFormat:@"CREATE INDEX `%@` on `%@`(`%@`)", idx[@"indexname"], self.name, idx[@"index"]] arguments:nil error:nil];
    //remove unnecessary indexes
    [oldIndexes enumerateKeysAndObjectsUsingBlock:^(id  _Nonnull key, id  _Nonnull obj, BOOL * _Nonnull stop) {
        //do not drop autoindex
        if(![key hasPrefix:@"sqlite_autoindex_"])[self.db executeUpdate:[NSString stringWithFormat:@"DROP INDEX `%@`", key] arguments:nil error:nil];
    }];
    return YES;
}

- (void)processedClassed:(NSMutableArray*)processed synchronizedClasses:(NSMutableArray*)synchronized{
    NSString *selfCls=NSStringFromClass(self.cls);
    [processed addObject:selfCls];
    if (self.isConformingToUniMM||self.isConformingToUniDB) {
        if (![synchronized containsObject:selfCls]) [synchronized addObject:selfCls];
    }
    if ([self.cls respondsToSelector:@selector(uni_synchronizedClasses)]){
        [[self.cls uni_synchronizedClasses] enumerateObjectsUsingBlock:^(id  _Nonnull obj, NSUInteger idx, BOOL * _Nonnull stop) {
            NSString *clsName=[obj isKindOfClass:NSString.class]?obj:NSStringFromClass(obj);
            if (![synchronized containsObject:clsName]) [synchronized addObject:clsName];
        }];
    }
    [self.propertyArr enumerateObjectsUsingBlock:^(UniProperty *  _Nonnull obj, NSUInteger idx, BOOL * _Nonnull stop) {
        if (!obj.cls) return;
        if ([processed containsObject:NSStringFromClass(obj.cls)]) return;
        UniClass *cls=[[UniClass alloc]initWithClass:obj.cls];
        [cls processedClassed:processed synchronizedClasses:synchronized];
    }];
}

- (NSArray*)synchronizedClasses{
    if (_synchronizedClasses) return _synchronizedClasses;
    NSMutableArray *processed=[NSMutableArray array];
    NSMutableArray *synchronized=[NSMutableArray array];
    [self processedClassed:processed synchronizedClasses:synchronized];
    _synchronizedClasses=synchronized;
    return _synchronizedClasses;
}

- (void)sync:(void(^)(void))block{
    static dispatch_once_t onceToken;
    static NSMapTable *mt;
    static dispatch_semaphore_t semaphore;
    dispatch_once(&onceToken, ^{
        mt=[NSMapTable strongToWeakObjectsMapTable];
        semaphore=dispatch_semaphore_create(1);
    });
    dispatch_semaphore_wait(semaphore, DISPATCH_TIME_FOREVER);
    NSRecursiveLock *lock;
    for (NSString * clsName in self.synchronizedClasses){
        lock=[mt objectForKey:clsName];
        if (lock) break;
    }
    if (!lock) lock=[[NSRecursiveLock alloc]init];
    for (NSString *clsName in self.synchronizedClasses) [mt setObject:lock forKey:clsName];
    dispatch_semaphore_signal(semaphore);
    [lock lock];
    for (NSString *clsName in self.synchronizedClasses){
        UniClass *cls=[UniClass classWithClass:NSClassFromString(clsName)];
        if (cls.db) [cls.db beginTransaction];
    }
    block();
    for (NSString *clsName in self.synchronizedClasses){
        UniClass *cls=[UniClass classWithClass:NSClassFromString(clsName)];
        if (cls.db) [cls.db commit];
    }
    [lock unlock];
}

- (BOOL)open:(NSString*)file error:(NSError* __autoreleasing *)error{
    __block BOOL suc;
    [self sync:^{
        [self.db close];
        suc=[self openAndInit:file error:error];
    }];
    return suc;
}

- (NSString*)description{
    NSMutableString *s=[NSMutableString string];
    for (UniProperty *property in self.jsonPropertyArr){
        NSMutableString *a=[NSMutableString string];
        [a appendString:@"["];
        for (NSArray *jsonKeyPath in property.jsonKeyPathArr) for (id keyPath in jsonKeyPath) [a appendFormat:@"%@.",keyPath];
        if (a.length>0) [a deleteCharactersInRange:NSMakeRange(a.length-1, 1)];
        [a appendString:@"]"];
        [s appendFormat:@"%@:%@,",property.name,a];
    }
    if (s.length>0) [s deleteCharactersInRange:NSMakeRange(s.length-1, 1)];
    NSMutableString *ss=[NSMutableString string];
    for (UniProperty *property in self.dbPropertyArr) [ss appendFormat:@"%@,",property.name];
    if (ss.length>0) [ss deleteCharactersInRange:NSMakeRange(ss.length-1, 1)];
    NSMutableString *r = [NSMutableString string];
    for (NSString * c in self.synchronizedClasses) [r appendFormat:@"%@,",c];
    if (r.length>0) [r deleteCharactersInRange:NSMakeRange(r.length-1, 1)];
    NSString * desc= [NSString stringWithFormat:@"****\n\
class name      : %@\n\
primary         : %@\n\
select sql      : %@\n\
insert sql      : %@\n\
update sql      : %@\n\
related classes : %@\n\
json_properties : %@\n\
db columns      : %@\n\
properties      : \n%@\n\
****",self.name,self.primaryProperty.name,self.dbSelectSql,self.dbInsertSql,self.dbUpdateSql,r,s,ss,self.propertyArr];
    desc=[desc stringByReplacingOccurrencesOfString:@"\\n" withString:@"\n"];
    return desc;
}

@end

@implementation UniProperty

- (instancetype)initWithProperty:(objc_property_t)property{
    self=[self init];
    if (!self) return nil;
    self.name = [NSString stringWithUTF8String:property_getName(property)];
    unsigned int count;
    objc_property_attribute_t *attrs = property_copyAttributeList(property, &count);
    for (int i = 0; i < count; i++) {
        objc_property_attribute_t attr=attrs[i];
        switch (attr.name[0]) {
            case 'T': {
                const char * v=attr.value;
                BOOL done=NO;
                do switch (v[0]) {
                    case 'r': self.methodEncoding |= UniMethodEncodingConst;v++;break;
                    case 'n': self.methodEncoding |= UniMethodEncodingIn;v++;break;
                    case 'N': self.methodEncoding |= UniMethodEncodingInout;v++;break;
                    case 'o': self.methodEncoding |= UniMethodEncodingOut;v++;break;
                    case 'O': self.methodEncoding |= UniMethodEncodingBycopy;v++;break;
                    case 'R': self.methodEncoding |= UniMethodEncodingByref;v++;break;
                    case 'V': self.methodEncoding |= UniMethodEncodingOneway;v++;break;
                    default: done=YES;break;
                }
                while(!done);
                switch (v[0]) {
                    case 'v': self.typeEncoding = UniTypeEncodingVoid;break;
                    case 'B': self.typeEncoding = UniTypeEncodingBool;break;
                    case 'c': self.typeEncoding = UniTypeEncodingInt8;break;
                    case 'C': self.typeEncoding = UniTypeEncodingUInt8;break;
                    case 's': self.typeEncoding = UniTypeEncodingInt16;break;
                    case 'S': self.typeEncoding = UniTypeEncodingUInt16;break;
                    case 'i': self.typeEncoding = UniTypeEncodingInt32;break;
                    case 'I': self.typeEncoding = UniTypeEncodingUInt32;break;
                    case 'l': self.typeEncoding = UniTypeEncodingInt32;break;
                    case 'L': self.typeEncoding = UniTypeEncodingUInt32;break;
                    case 'q': self.typeEncoding = UniTypeEncodingInt64;break;
                    case 'Q': self.typeEncoding = UniTypeEncodingUInt64;break;
                    case 'f': self.typeEncoding = UniTypeEncodingFloat;break;
                    case 'd': self.typeEncoding = UniTypeEncodingDouble;break;
                    case 'D': self.typeEncoding = UniTypeEncodingLongDouble;break;
                    case '#': self.typeEncoding = UniTypeEncodingClass;break;
                    case ':': self.typeEncoding = UniTypeEncodingSEL;break;
                    case '{': {
                        if(strcmp(v,@encode(NSRange))==0) self.typeEncoding=UniTypeEncodingNSRange;
                        else if(strcmp(v,@encode(NSPoint))==0) self.typeEncoding=UniTypeEncodingPoint;
                        else if(strcmp(v,@encode(NSSize))==0) self.typeEncoding=UniTypeEncodingSize;
                        else if(strcmp(v,@encode(NSRect))==0) self.typeEncoding=UniTypeEncodingRect;
                        else if(strcmp(v,@encode(NSEdgeInsets))==0) self.typeEncoding=UniTypeEncodingEdgeInsets;
#if TARGET_OS_IOS || TARGET_OS_TV
                        else if(strcmp(v,@encode(CATransform3D))==0) self.typeEncoding=UniTypeEncodingCATansform3D;
#endif
#if TARGET_OS_IOS || TARGET_OS_TV || TARGET_OS_WATCH
                        else if(strcmp(v,@encode(CGVector))==0) self.typeEncoding=UniTypeEncodingCGVector;
                        else if(strcmp(v,@encode(CGAffineTransform))==0) self.typeEncoding=UniTypeEncodingCGAffineTransform;
                        else if(strcmp(v,@encode(UIOffset))==0) self.typeEncoding=UniTypeEncodingUIOffset;
#endif
                        else{
#if TARGET_OS_IOS || TARGET_OS_TV || TARGET_OS_WATCH
                            if (@available(iOS 11.0, *)) {
                                if(@available(tvOS 11.0,*)){
                                    if (@available(watchOS 4.0, *)) {
                                        if(strcmp(v,@encode(NSDirectionalEdgeInsets))==0){
                                            self.typeEncoding=UniTypeEncodingNSDirectionalEdgeInsets;
                                            break;
                                        }
                                    }
                                }
                            }
#endif
                            self.typeEncoding=UniTypeEncodingStruct;
                        }
                    }break;
                    case '*': self.typeEncoding = UniTypeEncodingCString;break;
                    case '^': self.typeEncoding = UniTypeEncodingPointer;break;
                    case '[': self.typeEncoding = UniTypeEncodingCArray;break;
                    case '(': self.typeEncoding = UniTypeEncodingUnion;break;
                    case '@': {
                        if (strlen(v)==2&&v[1] =='?'){
                            self.typeEncoding = UniTypeEncodingBlock;
                            break;
                        }
                        NSScanner *scanner = [NSScanner scannerWithString:[NSString stringWithUTF8String:attr.value]];
                        if (![scanner scanString:@"@\"" intoString:NULL]) break;
                        NSString *clsName = nil;
                        if (![scanner scanUpToCharactersFromSet: [NSCharacterSet characterSetWithCharactersInString:@"\"<"] intoString:&clsName]||!clsName.length) break;
                        Class cls = NSClassFromString(clsName);
                        if (cls==NSString.class) self.typeEncoding=UniTypeEncodingNSString;
                        else if(cls==NSMutableString.class) self.typeEncoding=UniTypeEncodingNSMutableString;
                        else if(cls==NSURL.class) self.typeEncoding=UniTypeEncodingNSURL;
                        else if(cls==NSNumber.class) self.typeEncoding=UniTypeEncodingNSNumber;
                        else if(cls==NSDecimalNumber.class)self.typeEncoding=UniTypeEncodingNSDecimalNumber;
                        else if(cls==NSDate.class) self.typeEncoding=UniTypeEncodingNSDate;
                        else if(cls==NSData.class) self.typeEncoding=UniTypeEncodingNSData;
                        else if(cls==NSMutableData.class) self.typeEncoding=UniTypeEncodingNSMutableData;
                        else if(cls==NSOrderedSet.class) self.typeEncoding=UniTypeEncodingNSOrderedSet;
                        else if(cls==NSArray.class) self.typeEncoding=UniTypeEncodingNSArray;
                        else if(cls==NSMutableArray.class) self.typeEncoding=UniTypeEncodingNSMutableArray;
                        else if(cls==NSSet.class) self.typeEncoding=UniTypeEncodingNSSet;
                        else if(cls==NSMutableSet.class) self.typeEncoding=UniTypeEncodingNSMutableSet;
                        else if(cls==NSDictionary.class) self.typeEncoding=UniTypeEncodingNSDictionary;
                        else if(cls==NSMutableDictionary.class) self.typeEncoding=UniTypeEncodingNSMutableDictionary;
                        else self.typeEncoding=UniTypeEncodingNSObject;
                        self.cls=cls;
                    }
                    default: break;
                }
            } break;
            case 'V': break; //ivar
            case 'R': self.propertyEncoding|=UniPropertyEncodingReadonly; break;
            case 'C': self.propertyEncoding|=UniPropertyEncodingCopy; break;
            case '&': self.propertyEncoding|=UniPropertyEncodingRetain; break;
            case 'N': self.propertyEncoding|=UniPropertyEncodingNonatomic; break;
            case 'D': self.propertyEncoding|=UniPropertyEncodingDynamic; break;
            case 'W': self.propertyEncoding|=UniPropertyEncodingWeak; break;
            case 'G': {
                self.propertyEncoding|=UniPropertyEncodingCustomGetter;
                if (strlen(attr.value)) self.getter = NSSelectorFromString([NSString stringWithUTF8String:attrs[i].value]);
            } break;
            case 'S': {
                self.propertyEncoding|=UniPropertyEncodingCustomSetter;
                if (strlen(attr.value)) self.setter = NSSelectorFromString([NSString stringWithUTF8String:attrs[i].value]);
            } break;
            default: break;
        }
    }
    if (attrs) free(attrs);
    if (!self.getter) self.getter = NSSelectorFromString(self.name);
    if (!self.setter&&!(self.propertyEncoding&UniPropertyEncodingReadonly)) self.setter = NSSelectorFromString([NSString stringWithFormat:@"set%@%@:", [self.name substringToIndex:1].uppercaseString, [self.name substringFromIndex:1]]);
    return self;
}

- (NSString*)description{
    NSMutableString *s=[NSMutableString string];
    for (NSArray *jsonKeyPath in self.jsonKeyPathArr){
        for (NSString * keyPath in jsonKeyPath) [s appendFormat:@"%@.",keyPath];
        [s deleteCharactersInRange:NSMakeRange(s.length-1, 1)];
        [s appendString:@","];
    }
    [s deleteCharactersInRange:NSMakeRange(s.length-1, 1)];
    return [NSString stringWithFormat:@"\n\
    name                   : %@\n\
    encoding type          : %@\n\
    setter                 : %@\n\
    getter                 : %@\n\
    jsonKeyPaths           : %@\n\
    column type            : %@\n\
    json value transformer : %@\n\
    db value transformer   : %@\n    \
",self.name,uni_encodingDesc(self.typeEncoding,self.propertyEncoding,self.methodEncoding),NSStringFromSelector(self.setter),NSStringFromSelector(self.getter),s,uni_columnDesc(self.columnType),self.jsonTransformer,self.dbTransformer];
}
@end


text:
import { connect } from 'react-redux';
import { SignUp } from '../components/Account';
import { signUp, checkUsernameExist } from '../redux/account';

const mapStateToProps = (state) => ({
  signUpResult: state.account.signUpResult,
  checkUsernameExistResult: state.account.checkUsernameExistResult,
  isfetching: state.account.isfetching,
});

export default connect(
  mapStateToProps,
  { signUp, checkUsernameExist }
)(SignUp);


text:
package net.scalaleafs.test

import net.scalaleafs._
import akka.actor.ActorSystem
import scala.concurrent.ExecutionContext
import scala.concurrent.duration._

object Main extends App {

  implicit val actorSystem = ActorSystem("test")
  implicit val config = new Configuration(
    SprayServerContextPath -> List("bla")
  )

//  implicit val ec = actorSystem.dispatcher
//  for (i <- 0 to 40000000) {
//    val f = Future.successful(i)
//    val g = f.map(_ * 2)
//
//    result += 2
//  }
//  Thread.sleep(3000)
//  val start = System.currentTimeMillis
//  var result = 0
//  for (i <- 0 to 1000000) {
//    val f = Future.successful(i)
//    val g = f.map(_ * 2)
//
//    result += 2
//  }
//
//  println("G: " + result + " " + (System.currentTimeMillis - start) + " ms")
//

  val site = new Site(classOf[Frame], List("bla"), config)(actorSystem.dispatcher)
  val server = new SprayServer(classOf[Frame], config, actorSystem)
  server.start
}

case class SecurityContext(user : String)

//object SecurityContextVar extends ContextVar[SecurityContext]

class PageFrame(window : Window) extends Template {

  val url = CurrentUrl

  val render = bind(url) { u =>
    ".name" #> ("url:" + u) &
    "#clickme" #> setAttr("href", "println") &
    "#clickme" #> setText("click me not") &
    "#clickme" #> onclick(println("I HAVE BEEN CLICKED") & Noop) &
    ".content" #> bind(url) { url =>
      ""
    }
  }

  override val input =

Hi there, !

      click me

}


text:
'use strict';

var Page = require('./../start-process-modal');

module.exports = Page.extend({

  genericFormElement: function() {
    return element(by.css('.generic-form-fields'));
  },

  helpText: function() {
    return this.genericFormElement().element(by.css('.text-help')).getText();
  },

  businessKeyInput: function(inputValue) {
    var inputField = this.genericFormElement().
                      element(by.css('[cam-business-key]'));

    if (arguments.length !== 0)
      inputField.sendKeys(inputValue);

    return inputField;
  },

  addVariableButton: function() {
    return this.genericFormElement()
      .element(by.css('[ng-click="addVariable()"]'));
  },

  addVariable: function(name, type, value) {
    var that = this;

    this.addVariableButton().click();
    this.variableList().count().then(function(idx) {
      idx = idx -1;
      that.variableNameInput(idx, name);
      that.variableTypeInput(idx, type);
      if (type === 'Boolean') {
        if (value === 'true' || value === true)
          that.variableValueInput(idx).click();
      } else{
        that.variableValueInput(idx, value);
      };
    });
  },

  removeVariableButton: function(idx) {
    return this.genericFormElement()
      .element(by.css('[ng-click="removeVariable(delta)"]'));
  },

  removeVariable: function(idx) {
    this.removeVariableButton().click();
  },

  variableList: function() {
    return this.genericFormElement()
      .all(by.repeater('(delta, variable) in variables'));
  },

  variableInputElement: function(rowIdx, colIdx) {
    return this.variableList().get(rowIdx)
      .element(by.css('.row .col-xs-4:nth-child(' + colIdx + ')'));
  },

  variableInputField: function(rowIdx, colIdx, inputValue) {
    var inputField = this.variableInputElement(rowIdx, colIdx)
      .element(by.css('.form-control'));

    if (arguments.length === 3)
      inputField.sendKeys(inputValue);

    return inputField;
  },

  variableInputHelpText: function(rowIdx, colIdx) {
    return this.variableInputElement(rowIdx, colIdx)
      .element(by.css('.has-error .help-block')).getText();
  },

  isVariableInputFieldValide: function(rowIdx, colIdx) {
    return this.variableInputElement(rowIdx, colIdx)
      .getAttribute('class').then(function(classes) {
        return classes.indexOf('ng-valide') !== -1;
      })
  },

  variableNameInput: function(idx, inputValue) {
    return this.variableInputField(idx, 1, inputValue);
  },

  variableTypeInput: function(idx, inputValue) {
    return this.variableInputField(idx, 2, inputValue);
  },

  variableValueInput: function(idx, inputValue) {
    return this.variableInputField(idx, 3, inputValue);
  },

  variableNameHelpText: function(idx) {
    return this.variableInputHelpText(idx, 1);
  },

  variableValueHelpText: function(idx) {
    return this.variableInputHelpText(idx, 3);
  },

  isVariableNameInputValide: function(idx) {
    return this.isVariableInputFieldValide(idx, 1);
  },

  isVariableValueInputValide: function(idx) {
    return this.isVariableInputFieldValide(idx, 3);
  }

});


text: <header class="site-header"></header>