text:
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
    <modelVersion>4.0.0</modelVersion>

    <groupId>com.smartling</groupId>
    <artifactId>secor</artifactId>
    <version>0.26-SNAPSHOT</version>
    <packaging>jar</packaging>
    <name>secor</name>
    <description>Kafka to s3/gs/swift logs exporter</description>
    <url>https://github.com/smartling/secor</url>

    <licenses>
        <license>
            <name>Apache License, Version 2.0</name>
            <url>http://www.apache.org/licenses/LICENSE-2.0.txt</url>
            <distribution>repo</distribution>
            <comments>A business-friendly OSS license</comments>
        </license>
    </licenses>

    <developers>
        <developer>
            <id>pgarbacki</id>
            <name>Pawel Garbacki</name>
        </developer>
        <developer>
            <id>yuyang</id>
            <name>Yu Yang</name>
        </developer>
        <developer>
            <id>ramki</id>
            <name>Ramki Venkatachalam</name>
        </developer>
        <developer>
            <id>hcai</id>
            <name>Henry Cai</name>
        </developer>
    </developers>

    <scm>
        <connection>https://github.com/smartling/secor.git</connection>
        <developerConnection>https://github.com/smartling/secor.git</developerConnection>
        <url>https://github.com/smartling/secor</url>
    </scm>

    <properties>
        <maven.compiler.source>1.6</maven.compiler.source>
        <maven.compiler.target>1.6</maven.compiler.target>
        <encoding>UTF-8</encoding>
        <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
        <parquet.version>1.9.0</parquet.version>
    </properties>

    <distributionManagement>
        <repository>
            <id>ossrh</id>
            <url>https://oss.sonatype.org/service/local/staging/deploy/maven2/</url>
        </repository>
        <snapshotRepository>
            <id>ossrh</id>
            <url>https://oss.sonatype.org/content/repositories/snapshots</url>
        </snapshotRepository>
    </distributionManagement>

    <repositories>
        <repository>
            <id>Twitter public Maven repo</id>
            <url>http://maven.twttr.com</url>
        </repository>
        <repository>
            <id>Typesafe public Maven repo</id>
            <url>http://repo.typesafe.com/typesafe/releases</url>
        </repository>
    </repositories>
    <pluginRepositories>
        <pluginRepository>
            <id>Twitter public Maven repo</id>
            <url>http://maven.twttr.com</url>
        </pluginRepository>
        <pluginRepository>
            <id>Typesafe public Maven repo</id>
            <url>http://repo.typesafe.com/typesafe/releases</url>
        </pluginRepository>
    </pluginRepositories>

    <dependencies>
        <dependency>
            <groupId>com.google.protobuf</groupId>
            <artifactId>protobuf-java</artifactId>
            <version>3.1.0</version>
        </dependency>
        <dependency>
            <groupId>com.google.protobuf</groupId>
            <artifactId>protobuf-java-util</artifactId>
            <version>3.1.0</version>
        </dependency>
        <dependency>
            <groupId>com.amazonaws</groupId>
            <artifactId>aws-java-sdk-s3</artifactId>
            <version>1.11.160</version>
        </dependency>
        <dependency>
            <groupId>com.amazonaws</groupId>
            <artifactId>aws-java-sdk-sts</artifactId>
            <version>1.11.160</version>
        </dependency>
        <dependency>
            <groupId>net.java.dev.jets3t</groupId>
            <artifactId>jets3t</artifactId>
            <version>0.9.4</version>
        </dependency>
        <dependency>
            <groupId>log4j</groupId>
            <artifactId>log4j</artifactId>
            <version>1.2.17</version>
            <exclusions>
                <exclusion>
                    <groupId>com.sun.jmx</groupId>
                    <artifactId>jmxri</artifactId>
                </exclusion>
                <exclusion>
                    <groupId>com.sun.jdmk</groupId>
                    <artifactId>jmxtools</artifactId>
                </exclusion>
                <exclusion>
                    <groupId>javax.jms</groupId>
                    <artifactId>jms</artifactId>
                </exclusion>
            </exclusions>
        </dependency>
        <dependency>
            <groupId>org.slf4j</groupId>
            <artifactId>slf4j-api</artifactId>
            <version>1.7.2</version>
        </dependency>
        <dependency>
            <groupId>org.slf4j</groupId>
            <artifactId>jcl-over-slf4j</artifactId>
            <version>1.7.2</version>
        </dependency>
        <dependency>
            <groupId>org.slf4j</groupId>
            <artifactId>slf4j-log4j12</artifactId>
            <version>1.7.2</version>
        </dependency>
        <dependency>
            <groupId>commons-configuration</groupId>
            <artifactId>commons-configuration</artifactId>
            <version>1.9</version>
        </dependency>

        <dependency>
            <groupId>org.apache.hadoop</groupId>
            <artifactId>hadoop-common</artifactId>
            <version>2.7.0</version>
        </dependency>
        <dependency>
            <groupId>org.apache.hadoop</groupId>
            <artifactId>hadoop-mapreduce-client-core</artifactId>
            <version>2.7.0</version>
        </dependency>
        <dependency>
            <groupId>org.apache.hadoop</groupId>
            <artifactId>hadoop-aws</artifactId>
            <version>2.7.0</version>
            <exclusions>
                <exclusion>
                    <groupId>net.java.dev.jets3t</groupId>
                    <artifactId>jets3t</artifactId>
                </exclusion>
                <exclusion>
                    <groupId>com.fasterxml.jackson.core</groupId>
                    <artifactId>jackson-annotations</artifactId>
                </exclusion>
                <exclusion>
                    <groupId>com.fasterxml.jackson.core</groupId>
                    <artifactId>jackson-core</artifactId>
                </exclusion>
                <exclusion>
                    <groupId>com.fasterxml.jackson.core</groupId>
                    <artifactId>jackson-databind</artifactId>
                </exclusion>
                <exclusion>
                    <groupId>com.amazonaws</groupId>
                    <artifactId>aws-java-sdk</artifactId>
                </exclusion>
            </exclusions>
        </dependency>
        <dependency>
            <groupId>org.apache.hadoop</groupId>
            <artifactId>hadoop-openstack</artifactId>
            <version>2.7.0</version>
        </dependency>

        <!-- Parquet file format dependencies -->
        <dependency>
            <groupId>org.apache.parquet</groupId>
            <artifactId>parquet-common</artifactId>
            <version>${parquet.version}</version>
        </dependency>
        <dependency>
            <groupId>org.apache.parquet</groupId>
            <artifactId>parquet-encoding</artifactId>
            <version>${parquet.version}</version>
        </dependency>
        <dependency>
            <groupId>org.apache.parquet</groupId>
            <artifactId>parquet-column</artifactId>
            <version>${parquet.version}</version>
        </dependency>
        <dependency>
            <groupId>org.apache.parquet</groupId>
            <artifactId>parquet-hadoop</artifactId>
            <version>${parquet.version}</version>
        </dependency>
        <dependency>
            <groupId>org.apache.parquet</groupId>
            <artifactId>parquet-protobuf</artifactId>
            <version>${parquet.version}</version>
        </dependency>
        <dependency>
            <groupId>org.apache.parquet</groupId>
            <artifactId>parquet-thrift</artifactId>
            <version>${parquet.version}</version>
        </dependency>
        <dependency>
            <groupId>org.apache.thrift</groupId>
            <artifactId>libthrift</artifactId>
            <version>0.5.0</version>
        </dependency>
        <dependency>
            <groupId>com.twitter</groupId>
            <artifactId>ostrich_2.10</artifactId>
            <version>9.2.1</version>
        </dependency>
        <dependency>
            <groupId>com.twitter.common.zookeeper</groupId>
            <artifactId>lock</artifactId>
            <version>0.0.7</version>
        </dependency>
        <dependency>
            <groupId>com.google.guava</groupId>
            <artifactId>guava</artifactId>
            <version>18.0</version>
        </dependency>
        <dependency>
            <groupId>net.minidev</groupId>
            <artifactId>json-smart</artifactId>
            <version>2.0-RC3</version>
        </dependency>
        <dependency>
            <groupId>junit</groupId>
            <artifactId>junit</artifactId>
            <version>4.11</version>
            <scope>test</scope>
        </dependency>
        <dependency>
            <groupId>org.mockito</groupId>
            <artifactId>mockito-all</artifactId>
            <version>1.9.5</version>
        </dependency>
        <dependency>
            <groupId>org.powermock</groupId>
            <artifactId>powermock-module-junit4</artifactId>
            <version>1.5.2</version>
            <scope>test</scope>
        </dependency>
        <dependency>
            <groupId>org.powermock</groupId>
            <artifactId>powermock-api-mockito</artifactId>
            <version>1.5.2</version>
            <scope>test</scope>
        </dependency>
        <dependency>
            <groupId>org.msgpack</groupId>
            <artifactId>jackson-dataformat-msgpack</artifactId>
            <version>0.7.0-p2</version>
            <exclusions>
                <exclusion>
                    <groupId>com.fasterxml.jackson.core</groupId>
                    <artifactId>jackson-annotations</artifactId>
                </exclusion>
                <exclusion>
                    <groupId>com.fasterxml.jackson.core</groupId>
                    <artifactId>jackson-databind</artifactId>
                </exclusion>
            </exclusions>
        </dependency>
        <dependency>
            <groupId>com.timgroup</groupId>
            <artifactId>java-statsd-client</artifactId>
            <version>3.0.2</version>
        </dependency>
        <dependency>
            <groupId>com.google.apis</groupId>
            <artifactId>google-api-services-storage</artifactId>
            <version>v1-rev40-1.20.0</version>
            <exclusions>
                <exclusion>
                    <groupId>com.google.guava</groupId>
                    <artifactId>guava-jdk5</artifactId>
                </exclusion>
            </exclusions>
        </dependency>
        <dependency>
            <groupId>com.microsoft.azure</groupId>
            <artifactId>azure-storage</artifactId>
            <version>4.0.0</version>
        </dependency>
        <dependency>
            <groupId>org.apache.orc</groupId>
            <artifactId>orc-core</artifactId>
            <version>1.3.3</version>
        </dependency>
    </dependencies>

    <build>
        <resources>
            <resource>
                <directory>src/main/config</directory>
            </resource>
        </resources>
        <testResources>
            <testResource>
                <directory>src/test/config</directory>
            </testResource>
        </testResources>
        <plugins>
            <plugin>
                <groupId>org.codehaus.mojo</groupId>
                <artifactId>exec-maven-plugin</artifactId>
                <version>1.1</version>
                <executions>
                    <execution>
                        <phase>compile</phase>
                        <goals>
                            <goal>exec</goal>
                        </goals>
                        <configuration>
                            <executable>git</executable>
                            <arguments>
                                <argument>log</argument>
                                <argument>--pretty=format:build_revision=%H</argument>
                                <argument>-n1</argument>
                            </arguments>
                            <outputFile>target/build.properties</outputFile>
                        </configuration>
                    </execution>
                </executions>
            </plugin>
            <!-- This forces the dependencies to be copied to target/lib -->
            <plugin>
                <artifactId>maven-dependency-plugin</artifactId>
                <executions>
                    <execution>
                        <phase>package</phase>
                        <goals>
                            <goal>copy-dependencies</goal>
                        </goals>
                        <configuration>
                            <outputDirectory>${project.build.directory}/lib</outputDirectory>
                        </configuration>
                    </execution>
                </executions>
            </plugin>
            <plugin>
                <groupId>org.codehaus.mojo</groupId>
                <artifactId>exec-maven-plugin</artifactId>
                <version>1.1</version>
                <executions>
                    <execution>
                        <phase>compile</phase>
                        <goals>
                            <goal>exec</goal>
                        </goals>
                        <configuration>
                            <executable>git</executable>
                            <arguments>
                                <argument>log</argument>
                                <argument>--pretty=format:build_revision=%H</argument>
                                <argument>-n1</argument>
                            </arguments>
                            <outputFile>target/classes/com/pinterest/secor/common/build.properties</outputFile>
                        </configuration>
                    </execution>
                </executions>
            </plugin>
            <plugin>
                <groupId>org.apache.maven.plugins</groupId>
                <artifactId>maven-assembly-plugin</artifactId>
                <executions>
                    <execution>
                        <id>create-distribution</id>
                        <phase>package</phase>
                        <goals>
                            <goal>single</goal>
                        </goals>
                        <configuration>
                            <descriptors>
                                <descriptor>src/main/assembly/secor.xml</descriptor>
                            </descriptors>
                        </configuration>
                    </execution>
                </executions>
		    </plugin>
			<!-- Protobuf source generator plug-in -->
			<plugin>
				<groupId>com.github.os72</groupId>
				<artifactId>protoc-jar-maven-plugin</artifactId>
				<version>3.1.0</version>
				<executions>
					<execution>
						<id>protobuf-test-sources</id>
						<phase>generate-test-sources</phase>
						<goals>
							<goal>run</goal>
						</goals>
						<configuration>
							<protocVersion>3.1.0</protocVersion>
							<includeDirectories>
								<include>src/test/protobuf</include>
							</includeDirectories>
							<inputDirectories>
								<include>src/test/protobuf</include>
							</inputDirectories>
                            <includeStdTypes>true</includeStdTypes>
							<outputTargets>
								<outputTarget>
									<type>java</type>
									<addSources>none</addSources>
									<outputDirectory>target/generated-test-sources/protobuf/gen-java</outputDirectory>
								</outputTarget>
							</outputTargets>
						</configuration>
					</execution>
				</executions>
			</plugin>
			<plugin>
				<groupId>org.codehaus.mojo</groupId>
				<artifactId>build-helper-maven-plugin</artifactId>
				<version>1.10</version>
				<executions>
					<execution>
						<id>add-test-sources</id>
						<phase>generate-test-sources</phase>
						<goals>
							<goal>add-test-source</goal>
						</goals>
						<configuration>
							<sources>
								<source>target/generated-test-sources/protobuf/gen-java</source>
							</sources>
						</configuration>
					</execution>
				</executions>
			</plugin>
            <!-- Thrift source generate plugin. -->
            <plugin>
                <groupId>com.twitter</groupId>
                <artifactId>maven-finagle-thrift-plugin</artifactId>
                <version>0.0.9</version>
                <executions>
                    <execution>
                        <id>thrift-sources</id>
                        <phase>generate-sources</phase>
                        <goals>
                            <goal>compile</goal>
                        </goals>
                        <configuration>
                            <thriftGenerators>
                                <thriftGenerator>finagle</thriftGenerator>
                            </thriftGenerators>
                            <thriftOpts>
                                <thriftOpt>finagle</thriftOpt>
                            </thriftOpts>
                        </configuration>
                    </execution>
                    <execution>
                        <id>thrift-test-sources</id>
                        <phase>generate-test-sources</phase>
                        <goals>
                            <goal>testCompile</goal>
                        </goals>
                        <configuration>
                            <thriftGenerators>
                                <thriftGenerator>finagle</thriftGenerator>
                            </thriftGenerators>
                            <thriftOpts>
                                <thriftOpt>finagle</thriftOpt>
                            </thriftOpts>
                        </configuration>
                    </execution>
                </executions>
                <configuration>
                    <thriftGenerators>
                        <thriftGenerator>finagle</thriftGenerator>
                    </thriftGenerators>
                </configuration>
            </plugin>
            <plugin>
                <groupId>org.codehaus.mojo</groupId>
                <artifactId>license-maven-plugin</artifactId>
                <version>1.6</version>
                <executions>
                    <execution>
                        <id>download-licenses</id>
                        <goals>
                            <goal>download-licenses</goal>
                        </goals>
                    </execution>
                </executions>
            </plugin>
            <plugin>
                <groupId>org.apache.maven.plugins</groupId>
                <artifactId>maven-jar-plugin</artifactId>
                <version>2.4</version>
                <executions>
                    <execution>
                        <goals>
                            <goal>test-jar</goal>
                        </goals>
                    </execution>
                </executions>
            </plugin>
            <plugin>
                <groupId>org.sonatype.plugins</groupId>
                <artifactId>nexus-staging-maven-plugin</artifactId>
                <version>1.6.3</version>
                <extensions>true</extensions>
                <configuration>
                    <serverId>ossrh</serverId>
                    <nexusUrl>https://oss.sonatype.org/</nexusUrl>
                    <autoReleaseAfterClose>true</autoReleaseAfterClose>
                </configuration>
            </plugin>
            <plugin>
                <groupId>org.apache.maven.plugins</groupId>
                <artifactId>maven-compiler-plugin</artifactId>
                <configuration>
                    <source>1.7</source>
                    <target>1.7</target>
                </configuration>
            </plugin>
        </plugins>
    </build>

    <profiles>
        <profile>
            <id>release</id>
            <build>
                <plugins>
                    <plugin>
                        <groupId>org.apache.maven.plugins</groupId>
                        <artifactId>maven-source-plugin</artifactId>
                        <version>2.2.1</version>
                        <executions>
                            <execution>
                                <id>attach-sources</id>
                                <goals>
                                    <goal>jar-no-fork</goal>
                                </goals>
                              </execution>
                          </executions>
                    </plugin>
                    <plugin>
                        <groupId>org.apache.maven.plugins</groupId>
                        <artifactId>maven-javadoc-plugin</artifactId>
                        <version>2.9.1</version>
                        <executions>
                            <execution>
                                <id>attach-javadocs</id>
                                <goals>
                                    <goal>jar</goal>
                                </goals>
                              </execution>
                          </executions>
                    </plugin>
                    <plugin>
                      <groupId>org.apache.maven.plugins</groupId>
                      <artifactId>maven-gpg-plugin</artifactId>
                      <version>1.5</version>
                      <executions>
                          <execution>
                              <id>sign-artifacts</id>
                              <phase>verify</phase>
                              <goals>
                                  <goal>sign</goal>
                              </goals>
                            </execution>
                        </executions>
                    </plugin>
                </plugins>
            </build>
            <dependencies>
                <dependency>
                    <groupId>org.apache.kafka</groupId>
                    <artifactId>kafka_2.10</artifactId>
                    <version>0.10.2.0</version>
                    <exclusions>
                        <exclusion>
                            <groupId>org.slf4j</groupId>
                            <artifactId>slf4j-simple</artifactId>
                        </exclusion>
                    </exclusions>
                </dependency>
            </dependencies>
        </profile>
        <profile>
            <id>kafka-0.8-dev</id>
            <build>
                <plugins>
                    <plugin>
                        <groupId>org.apache.maven.plugins</groupId>
                        <artifactId>maven-compiler-plugin</artifactId>
                        <version>2.0.2</version>
                        <configuration>
                            <excludes>
                                <exclude>com/pinterest/secor/timestamp/Kafka10MessageTimestamp.java</exclude>
                            </excludes>
                        </configuration>
                    </plugin>
                </plugins>
            </build>
            <dependencies>
                <dependency>
                    <groupId>org.apache.kafka</groupId>
                    <artifactId>kafka_2.10</artifactId>
                    <version>0.8.2.1</version>
                    <exclusions>
                        <exclusion>
                            <groupId>org.slf4j</groupId>
                            <artifactId>slf4j-simple</artifactId>
                        </exclusion>
                    </exclusions>
                </dependency>
            </dependencies>
        </profile>
        <profile>
            <id>kafka-0.10-dev</id>
            <activation>
                <activeByDefault>true</activeByDefault>
            </activation>
            <dependencies>
                <dependency>
                    <groupId>org.apache.kafka</groupId>
                    <artifactId>kafka_2.10</artifactId>
                    <version>0.10.2.0</version>
                    <exclusions>
                        <exclusion>
                            <groupId>org.slf4j</groupId>
                            <artifactId>slf4j-simple</artifactId>
                        </exclusion>
                    </exclusions>
                </dependency>
            </dependencies>
        </profile>
    </profiles>
</project>


text:
from soap import logger

from soap.common import indent
from soap.expression import (
    AccessExpr, Expression, FixExpr, OutputVariableTuple, SelectExpr,
    UpdateExpr, Variable
)
from soap.semantics.common import is_numeral
from soap.semantics.error import cast
from soap.semantics.functions import expand_expr
from soap.semantics.label import Label
from soap.semantics.state.base import BaseState

class MetaState(BaseState, dict):
    __slots__ = ('_hash')

    def __init__(self, dictionary=None, **kwargs):
        dictionary = dict(dictionary or {}, **kwargs)
        mapping = {
            self._cast_key(key): self._cast_value(key, value)
            for key, value in dictionary.items()
        }
        super().__init__(mapping)
        self._hash = None

    def _cast_key(self, key):
        if isinstance(key, (Variable, Label, OutputVariableTuple)):
            return key
        if isinstance(key, str):
            var_list = [var for var in self.keys() if var.name == key]
            if not var_list:
                raise KeyError(key)
            if len(var_list) > 1:
                raise KeyError('Multiple variables with the same name.')
            var = var_list.pop()
            return var
        raise TypeError(
            'Do not know how to convert {!r} into a variable'.format(key))

    def _cast_value(self, key, value):
        if isinstance(value, (Label, Expression)):
            return value
        if isinstance(value, str):
            from soap.parser import parse
            return parse(value)
        if isinstance(value, (int, float)) or is_numeral(value):
            return cast(value)
        if isinstance(value, dict):
            return self.__class__(value)
        raise TypeError(
            'Do not know how to convert {!r} into an expression'.format(value))

    def __getitem__(self, key):
        try:
            return super().__getitem__(key)
        except KeyError:
            logger.warning('Variable {} does not exist.'.format(key))
            return Variable('__unreachable', key.dtype)

    def is_fixpoint(self, other):
        raise RuntimeError('Should not be called.')

    def widen(self, other):
        raise RuntimeError('Should not be called.')

    def visit_SkipFlow(self, flow):
        return self

    def visit_AssignFlow(self, flow):
        var, expr = flow.var, flow.expr
        if isinstance(var, AccessExpr):
            var, subscript = var.var, var.subscript
            expr = UpdateExpr(var, subscript, expr)
        return self.immu_update(var, expand_expr(expr, self))

    def visit_IfFlow(self, flow):
        def get(state, var):
            expr = state[var]
            return expr
        bool_expr = expand_expr(flow.conditional_expr, self)
        true_state = self.transition(flow.true_flow)
        false_state = self.transition(flow.false_flow)
        var_list = set(self.keys())
        var_list |= set(true_state.keys()) | set(false_state.keys())
        mapping = dict(self)
        for var in var_list:
            true_expr = get(true_state, var)
            false_expr = get(false_state, var)
            if true_expr == false_expr:
                value = true_expr
            else:
                value = SelectExpr(bool_expr, true_expr, false_expr)
            mapping[var] = value
        return self.__class__(mapping)

    @staticmethod
    def _input_vars(meta_state, var):
        in_vars = {var}
        next_vars = {var}
        while next_vars:
            var = next_vars.pop()
            expr = meta_state.get(var)
            if expr is None or is_numeral(expr):
                continue
            expr_vars = expr.vars()
            next_vars |= expr_vars - in_vars
            in_vars |= expr_vars
        return in_vars

    def _visit_loop(self, init_state, bool_expr, loop_flow):
        """
        Finds necessary loop variables and loop states for each variable.
        """
        bool_expr_vars = bool_expr.vars()
        input_vars = loop_flow.vars(output=False) | bool_expr_vars
        loop_vars = loop_flow.vars(input=False)
        id_state = self.__class__({k: k for k in input_vars | loop_vars})
        loop_state = id_state.transition(loop_flow)

        loop_map = {}
        init_map = {}

        for var in loop_vars:
            # local loop/init variables
            local_loop_vars = self._input_vars(loop_state, var)
            local_loop_vars |= bool_expr_vars
            # local loop/init states
            loop_map[var] = self.__class__(
                {k: v for k, v in loop_state.items() if k in local_loop_vars})
            init_map[var] = self.__class__(
                {k: v for k, v in init_state.items() if k in local_loop_vars})

        mapping = dict(init_state)
        for var in loop_map:
            # fixpoint expression
            mapping[var] = FixExpr(
                bool_expr, loop_map[var], var, init_map[var])
        return self.__class__(mapping)

    def visit_WhileFlow(self, flow):
        return self._visit_loop(self, flow.conditional_expr, flow.loop_flow)

    def visit_ForFlow(self, flow):
        init_state = self(flow.init_flow)
        return self._visit_loop(
            init_state, flow.conditional_expr, flow.loop_flow + flow.incr_flow)

    def format(self):
        items = []
        for k, v in sorted(self.items(), key=str):
            if isinstance(v, (Expression, MetaState)):
                v = v.format()
            items.append('{}: {}'.format(k, v))
        items = ', \n'.join(items)
        return '{{\n{}}}'.format(indent(items))

    def __str__(self):
        return self.format().replace('    ', '').replace('\n', '').strip()

    def __hash__(self):
        if self._hash is None:
            self._hash = hash(tuple(sorted(self.items(), key=hash)))
        return self._hash

def flow_to_meta_state(flow):
    id_state = MetaState({v: v for v in flow.vars(output=False)})
    return id_state.transition(flow)

redpajama_set_name: RedPajamaGithub


text:
#ifndef __CPU_O3_LSQ_UNIT_IMPL_HH__
#define __CPU_O3_LSQ_UNIT_IMPL_HH__

#include "arch/generic/debugfaults.hh"
#include "arch/locked_mem.hh"
#include "base/str.hh"
#include "config/the_isa.hh"
#include "cpu/checker/cpu.hh"
#include "cpu/o3/lsq.hh"
#include "cpu/o3/lsq_unit.hh"
#include "debug/Activity.hh"
#include "debug/IEW.hh"
#include "debug/LSQUnit.hh"
#include "debug/O3PipeView.hh"
#include "mem/packet.hh"
#include "mem/request.hh"

template<class Impl>
LSQUnit<Impl>::WritebackEvent::WritebackEvent(DynInstPtr &_inst, PacketPtr _pkt,
                                              LSQUnit *lsq_ptr)
    : Event(Default_Pri, AutoDelete),
      inst(_inst), pkt(_pkt), lsqPtr(lsq_ptr)
{
}

template<class Impl>
void
LSQUnit<Impl>::WritebackEvent::process()
{
    assert(!lsqPtr->cpu->switchedOut());

    lsqPtr->writeback(inst, pkt);

    if (pkt->senderState)
        delete pkt->senderState;

    delete pkt->req;
    delete pkt;
}

template<class Impl>
const char *
LSQUnit<Impl>::WritebackEvent::description() const
{
    return "Store writeback";
}

template<class Impl>
void
LSQUnit<Impl>::completeDataAccess(PacketPtr pkt)
{
    LSQSenderState *state = dynamic_cast<LSQSenderState *>(pkt->senderState);
    DynInstPtr inst = state->inst;
    DPRINTF(IEW, "Writeback event [sn:%lli].\n", inst->seqNum);
    DPRINTF(Activity, "Activity: Writeback event [sn:%lli].\n", inst->seqNum);

    if (state->cacheBlocked) {
        // This is the first half of a previous split load,
        // where the 2nd half blocked, ignore this response
        DPRINTF(IEW, "[sn:%lli]: Response from first half of earlier "
                "blocked split load recieved. Ignoring.\n", inst->seqNum);
        delete state;
        return;
    }

    // If this is a split access, wait until all packets are received.
    if (TheISA::HasUnalignedMemAcc && !state->complete()) {
        return;
    }

    assert(!cpu->switchedOut());
    if (!inst->isSquashed()) {
        if (!state->noWB) {
            if (!TheISA::HasUnalignedMemAcc || !state->isSplit ||
                !state->isLoad) {
                writeback(inst, pkt);
            } else {
                writeback(inst, state->mainPkt);
            }
        }

        if (inst->isStore()) {
            completeStore(state->idx);
        }
    }

    if (TheISA::HasUnalignedMemAcc && state->isSplit && state->isLoad) {
        delete state->mainPkt->req;
        delete state->mainPkt;
    }

    pkt->req->setAccessLatency();
    cpu->ppDataAccessComplete->notify(std::make_pair(inst, pkt));

    delete state;
}

template <class Impl>
LSQUnit<Impl>::LSQUnit()
    : loads(0), stores(0), storesToWB(0), cacheBlockMask(0), stalled(false),
      isStoreBlocked(false), storeInFlight(false), hasPendingPkt(false)
{
}

template<class Impl>
void
LSQUnit<Impl>::init(O3CPU *cpu_ptr, IEW *iew_ptr, DerivO3CPUParams *params,
        LSQ *lsq_ptr, unsigned maxLQEntries, unsigned maxSQEntries,
        unsigned id)
{
    cpu = cpu_ptr;
    iewStage = iew_ptr;

    lsq = lsq_ptr;

    lsqID = id;

    DPRINTF(LSQUnit, "Creating LSQUnit%i object.\n",id);

    // Add 1 for the sentinel entry (they are circular queues).
    LQEntries = maxLQEntries + 1;
    SQEntries = maxSQEntries + 1;

    //Due to uint8_t index in LSQSenderState
    assert(LQEntries <= 256);
    assert(SQEntries <= 256);

    loadQueue.resize(LQEntries);
    storeQueue.resize(SQEntries);

    depCheckShift = params->LSQDepCheckShift;
    checkLoads = params->LSQCheckLoads;
    cachePorts = params->cachePorts;
    needsTSO = params->needsTSO;

    resetState();
}

template<class Impl>
void
LSQUnit<Impl>::resetState()
{
    loads = stores = storesToWB = 0;

    loadHead = loadTail = 0;

    storeHead = storeWBIdx = storeTail = 0;

    usedPorts = 0;

    retryPkt = NULL;
    memDepViolator = NULL;

    stalled = false;

    cacheBlockMask = ~(cpu->cacheLineSize() - 1);
}

template<class Impl>
std::string
LSQUnit<Impl>::name() const
{
    if (Impl::MaxThreads == 1) {
        return iewStage->name() + ".lsq";
    } else {
        return iewStage->name() + ".lsq.thread" + std::to_string(lsqID);
    }
}

template<class Impl>
void
LSQUnit<Impl>::regStats()
{
    lsqForwLoads
        .name(name() + ".forwLoads")
        .desc("Number of loads that had data forwarded from stores");

    invAddrLoads
        .name(name() + ".invAddrLoads")
        .desc("Number of loads ignored due to an invalid address");

    lsqSquashedLoads
        .name(name() + ".squashedLoads")
        .desc("Number of loads squashed");

    lsqIgnoredResponses
        .name(name() + ".ignoredResponses")
        .desc("Number of memory responses ignored because the instruction is squashed");

    lsqMemOrderViolation
        .name(name() + ".memOrderViolation")
        .desc("Number of memory ordering violations");

    lsqSquashedStores
        .name(name() + ".squashedStores")
        .desc("Number of stores squashed");

    invAddrSwpfs
        .name(name() + ".invAddrSwpfs")
        .desc("Number of software prefetches ignored due to an invalid address");

    lsqBlockedLoads
        .name(name() + ".blockedLoads")
        .desc("Number of blocked loads due to partial load-store forwarding");

    lsqRescheduledLoads
        .name(name() + ".rescheduledLoads")
        .desc("Number of loads that were rescheduled");

    lsqCacheBlocked
        .name(name() + ".cacheBlocked")
        .desc("Number of times an access to memory failed due to the cache being blocked");
}

template<class Impl>
void
LSQUnit<Impl>::setDcachePort(MasterPort *dcache_port)
{
    dcachePort = dcache_port;
}

template<class Impl>
void
LSQUnit<Impl>::clearLQ()
{
    loadQueue.clear();
}

template<class Impl>
void
LSQUnit<Impl>::clearSQ()
{
    storeQueue.clear();
}

template<class Impl>
void
LSQUnit<Impl>::drainSanityCheck() const
{
    for (int i = 0; i < loadQueue.size(); ++i)
        assert(!loadQueue[i]);

    assert(storesToWB == 0);
    assert(!retryPkt);
}

template<class Impl>
void
LSQUnit<Impl>::takeOverFrom()
{
    resetState();
}

template<class Impl>
void
LSQUnit<Impl>::resizeLQ(unsigned size)
{
    unsigned size_plus_sentinel = size + 1;
    assert(size_plus_sentinel >= LQEntries);

    if (size_plus_sentinel > LQEntries) {
        while (size_plus_sentinel > loadQueue.size()) {
            DynInstPtr dummy;
            loadQueue.push_back(dummy);
            LQEntries++;
        }
    } else {
        LQEntries = size_plus_sentinel;
    }

    assert(LQEntries <= 256);
}

template<class Impl>
void
LSQUnit<Impl>::resizeSQ(unsigned size)
{
    unsigned size_plus_sentinel = size + 1;
    if (size_plus_sentinel > SQEntries) {
        while (size_plus_sentinel > storeQueue.size()) {
            SQEntry dummy;
            storeQueue.push_back(dummy);
            SQEntries++;
        }
    } else {
        SQEntries = size_plus_sentinel;
    }

    assert(SQEntries <= 256);
}

template <class Impl>
void
LSQUnit<Impl>::insert(DynInstPtr &inst)
{
    assert(inst->isMemRef());

    assert(inst->isLoad() || inst->isStore());

    if (inst->isLoad()) {
        insertLoad(inst);
    } else {
        insertStore(inst);
    }

    inst->setInLSQ();
}

template <class Impl>
void
LSQUnit<Impl>::insertLoad(DynInstPtr &load_inst)
{
    assert((loadTail + 1) % LQEntries != loadHead);
    assert(loads < LQEntries);

    DPRINTF(LSQUnit, "Inserting load PC %s, idx:%i [sn:%lli]\n",
            load_inst->pcState(), loadTail, load_inst->seqNum);

    load_inst->lqIdx = loadTail;

    if (stores == 0) {
        load_inst->sqIdx = -1;
    } else {
        load_inst->sqIdx = storeTail;
    }

    loadQueue[loadTail] = load_inst;

    incrLdIdx(loadTail);

    ++loads;
}

template <class Impl>
void
LSQUnit<Impl>::insertStore(DynInstPtr &store_inst)
{
    // Make sure it is not full before inserting an instruction.
    assert((storeTail + 1) % SQEntries != storeHead);
    assert(stores < SQEntries);

    DPRINTF(LSQUnit, "Inserting store PC %s, idx:%i [sn:%lli]\n",
            store_inst->pcState(), storeTail, store_inst->seqNum);

    store_inst->sqIdx = storeTail;
    store_inst->lqIdx = loadTail;

    storeQueue[storeTail] = SQEntry(store_inst);

    incrStIdx(storeTail);

    ++stores;
}

template <class Impl>
typename Impl::DynInstPtr
LSQUnit<Impl>::getMemDepViolator()
{
    DynInstPtr temp = memDepViolator;

    memDepViolator = NULL;

    return temp;
}

template <class Impl>
unsigned
LSQUnit<Impl>::numFreeLoadEntries()
{
        //LQ has an extra dummy entry to differentiate
        //empty/full conditions. Subtract 1 from the free entries.
        DPRINTF(LSQUnit, "LQ size: %d, #loads occupied: %d\n", LQEntries, loads);
        return LQEntries - loads - 1;
}

template <class Impl>
unsigned
LSQUnit<Impl>::numFreeStoreEntries()
{
        //SQ has an extra dummy entry to differentiate
        //empty/full conditions. Subtract 1 from the free entries.
        DPRINTF(LSQUnit, "SQ size: %d, #stores occupied: %d\n", SQEntries, stores);
        return SQEntries - stores - 1;

 }

template <class Impl>
void
LSQUnit<Impl>::checkSnoop(PacketPtr pkt)
{
    int load_idx = loadHead;
    DPRINTF(LSQUnit, "Got snoop for address %#x\n", pkt->getAddr());

    // Unlock the cpu-local monitor when the CPU sees a snoop to a locked
    // address. The CPU can speculatively execute a LL operation after a pending
    // SC operation in the pipeline and that can make the cache monitor the CPU
    // is connected to valid while it really shouldn't be.
    for (int x = 0; x < cpu->numContexts(); x++) {
        ThreadContext *tc = cpu->getContext(x);
        bool no_squash = cpu->thread[x]->noSquashFromTC;
        cpu->thread[x]->noSquashFromTC = true;
        TheISA::handleLockedSnoop(tc, pkt, cacheBlockMask);
        cpu->thread[x]->noSquashFromTC = no_squash;
    }

    Addr invalidate_addr = pkt->getAddr() & cacheBlockMask;

    DynInstPtr ld_inst = loadQueue[load_idx];
    if (ld_inst) {
        Addr load_addr_low = ld_inst->physEffAddrLow & cacheBlockMask;
        Addr load_addr_high = ld_inst->physEffAddrHigh & cacheBlockMask;

        // Check that this snoop didn't just invalidate our lock flag
        if (ld_inst->effAddrValid() && (load_addr_low == invalidate_addr
                                        || load_addr_high == invalidate_addr)
            && ld_inst->memReqFlags & Request::LLSC)
            TheISA::handleLockedSnoopHit(ld_inst.get());
    }

    // If this is the only load in the LSQ we don't care
    if (load_idx == loadTail)
        return;

    incrLdIdx(load_idx);

    bool force_squash = false;

    while (load_idx != loadTail) {
        DynInstPtr ld_inst = loadQueue[load_idx];

        if (!ld_inst->effAddrValid() || ld_inst->strictlyOrdered()) {
            incrLdIdx(load_idx);
            continue;
        }

        Addr load_addr_low = ld_inst->physEffAddrLow & cacheBlockMask;
        Addr load_addr_high = ld_inst->physEffAddrHigh & cacheBlockMask;

        DPRINTF(LSQUnit, "-- inst [sn:%lli] load_addr: %#x to pktAddr:%#x\n",
                    ld_inst->seqNum, load_addr_low, invalidate_addr);

        if ((load_addr_low == invalidate_addr
             || load_addr_high == invalidate_addr) || force_squash) {
            if (needsTSO) {
                // If we have a TSO system, as all loads must be ordered with
                // all other loads, this load as well as *all* subsequent loads
                // need to be squashed to prevent possible load reordering.
                force_squash = true;
            }
            if (ld_inst->possibleLoadViolation() || force_squash) {
                DPRINTF(LSQUnit, "Conflicting load at addr %#x [sn:%lli]\n",
                        pkt->getAddr(), ld_inst->seqNum);

                // Mark the load for re-execution
                ld_inst->fault = std::make_shared<ReExec>();
            } else {
                DPRINTF(LSQUnit, "HitExternal Snoop for addr %#x [sn:%lli]\n",
                        pkt->getAddr(), ld_inst->seqNum);

                // Make sure that we don't lose a snoop hitting a LOCKED
                // address since the LOCK* flags don't get updated until
                // commit.
                if (ld_inst->memReqFlags & Request::LLSC)
                    TheISA::handleLockedSnoopHit(ld_inst.get());

                // If a older load checks this and it's true
                // then we might have missed the snoop
                // in which case we need to invalidate to be sure
                ld_inst->hitExternalSnoop(true);
            }
        }
        incrLdIdx(load_idx);
    }
    return;
}

template <class Impl>
Fault
LSQUnit<Impl>::checkViolations(int load_idx, DynInstPtr &inst)
{
    Addr inst_eff_addr1 = inst->effAddr >> depCheckShift;
    Addr inst_eff_addr2 = (inst->effAddr + inst->effSize - 1) >> depCheckShift;

    /** @todo in theory you only need to check an instruction that has executed
     * however, there isn't a good way in the pipeline at the moment to check
     * all instructions that will execute before the store writes back. Thus,
     * like the implementation that came before it, we're overly conservative.
     */
    while (load_idx != loadTail) {
        DynInstPtr ld_inst = loadQueue[load_idx];
        if (!ld_inst->effAddrValid() || ld_inst->strictlyOrdered()) {
            incrLdIdx(load_idx);
            continue;
        }

        Addr ld_eff_addr1 = ld_inst->effAddr >> depCheckShift;
        Addr ld_eff_addr2 =
            (ld_inst->effAddr + ld_inst->effSize - 1) >> depCheckShift;

        if (inst_eff_addr2 >= ld_eff_addr1 && inst_eff_addr1 <= ld_eff_addr2) {
            if (inst->isLoad()) {
                // If this load is to the same block as an external snoop
                // invalidate that we've observed then the load needs to be
                // squashed as it could have newer data
                if (ld_inst->hitExternalSnoop()) {
                    if (!memDepViolator ||
                            ld_inst->seqNum < memDepViolator->seqNum) {
                        DPRINTF(LSQUnit, "Detected fault with inst [sn:%lli] "
                                "and [sn:%lli] at address %#x\n",
                                inst->seqNum, ld_inst->seqNum, ld_eff_addr1);
                        memDepViolator = ld_inst;

                        ++lsqMemOrderViolation;

                        return std::make_shared<GenericISA::M5PanicFault>(
                            "Detected fault with inst [sn:%lli] and "
                            "[sn:%lli] at address %#x\n",
                            inst->seqNum, ld_inst->seqNum, ld_eff_addr1);
                    }
                }

                // Otherwise, mark the load has a possible load violation
                // and if we see a snoop before it's commited, we need to squash
                ld_inst->possibleLoadViolation(true);
                DPRINTF(LSQUnit, "Found possible load violation at addr: %#x"
                        " between instructions [sn:%lli] and [sn:%lli]\n",
                        inst_eff_addr1, inst->seqNum, ld_inst->seqNum);
            } else {
                // A load/store incorrectly passed this store.
                // Check if we already have a violator, or if it's newer
                // squash and refetch.
                if (memDepViolator && ld_inst->seqNum > memDepViolator->seqNum)
                    break;

                DPRINTF(LSQUnit, "Detected fault with inst [sn:%lli] and "
                        "[sn:%lli] at address %#x\n",
                        inst->seqNum, ld_inst->seqNum, ld_eff_addr1);
                memDepViolator = ld_inst;

                ++lsqMemOrderViolation;

                return std::make_shared<GenericISA::M5PanicFault>(
                    "Detected fault with "
                    "inst [sn:%lli] and [sn:%lli] at address %#x\n",
                    inst->seqNum, ld_inst->seqNum, ld_eff_addr1);
            }
        }

        incrLdIdx(load_idx);
    }
    return NoFault;
}

template <class Impl>
Fault
LSQUnit<Impl>::executeLoad(DynInstPtr &inst)
{
    using namespace TheISA;
    // Execute a specific load.
    Fault load_fault = NoFault;

    DPRINTF(LSQUnit, "Executing load PC %s, [sn:%lli]\n",
            inst->pcState(), inst->seqNum);

    assert(!inst->isSquashed());

    load_fault = inst->initiateAcc();

    if (inst->isTranslationDelayed() &&
        load_fault == NoFault)
        return load_fault;

    // If the instruction faulted or predicated false, then we need to send it
    // along to commit without the instruction completing.
    if (load_fault != NoFault || !inst->readPredicate()) {
        // Send this instruction to commit, also make sure iew stage
        // realizes there is activity.  Mark it as executed unless it
        // is a strictly ordered load that needs to hit the head of
        // commit.
        if (!inst->readPredicate())
            inst->forwardOldRegs();
        DPRINTF(LSQUnit, "Load [sn:%lli] not executed from %s\n",
                inst->seqNum,
                (load_fault != NoFault ? "fault" : "predication"));
        if (!(inst->hasRequest() && inst->strictlyOrdered()) ||
            inst->isAtCommit()) {
            inst->setExecuted();
        }
        iewStage->instToCommit(inst);
        iewStage->activityThisCycle();
    } else {
        assert(inst->effAddrValid());
        int load_idx = inst->lqIdx;
        incrLdIdx(load_idx);

        if (checkLoads)
            return checkViolations(load_idx, inst);
    }

    return load_fault;
}

template <class Impl>
Fault
LSQUnit<Impl>::executeStore(DynInstPtr &store_inst)
{
    using namespace TheISA;
    // Make sure that a store exists.
    assert(stores != 0);

    int store_idx = store_inst->sqIdx;

    DPRINTF(LSQUnit, "Executing store PC %s [sn:%lli]\n",
            store_inst->pcState(), store_inst->seqNum);

    assert(!store_inst->isSquashed());

    // Check the recently completed loads to see if any match this store's
    // address.  If so, then we have a memory ordering violation.
    int load_idx = store_inst->lqIdx;

    Fault store_fault = store_inst->initiateAcc();

    if (store_inst->isTranslationDelayed() &&
        store_fault == NoFault)
        return store_fault;

    if (!store_inst->readPredicate())
        store_inst->forwardOldRegs();

    if (storeQueue[store_idx].size == 0) {
        DPRINTF(LSQUnit,"Fault on Store PC %s, [sn:%lli], Size = 0\n",
                store_inst->pcState(), store_inst->seqNum);

        return store_fault;
    } else if (!store_inst->readPredicate()) {
        DPRINTF(LSQUnit, "Store [sn:%lli] not executed from predication\n",
                store_inst->seqNum);
        return store_fault;
    }

    assert(store_fault == NoFault);

    if (store_inst->isStoreConditional()) {
        // Store conditionals need to set themselves as able to
        // writeback if we haven't had a fault by here.
        storeQueue[store_idx].canWB = true;

        ++storesToWB;
    }

    return checkViolations(load_idx, store_inst);

}

template <class Impl>
void
LSQUnit<Impl>::commitLoad()
{
    assert(loadQueue[loadHead]);

    DPRINTF(LSQUnit, "Committing head load instruction, PC %s\n",
            loadQueue[loadHead]->pcState());

    loadQueue[loadHead] = NULL;

    incrLdIdx(loadHead);

    --loads;
}

template <class Impl>
void
LSQUnit<Impl>::commitLoads(InstSeqNum &youngest_inst)
{
    assert(loads == 0 || loadQueue[loadHead]);

    while (loads != 0 && loadQueue[loadHead]->seqNum <= youngest_inst) {
        commitLoad();
    }
}

template <class Impl>
void
LSQUnit<Impl>::commitStores(InstSeqNum &youngest_inst)
{
    assert(stores == 0 || storeQueue[storeHead].inst);

    int store_idx = storeHead;

    while (store_idx != storeTail) {
        assert(storeQueue[store_idx].inst);
        // Mark any stores that are now committed and have not yet
        // been marked as able to write back.
        if (!storeQueue[store_idx].canWB) {
            if (storeQueue[store_idx].inst->seqNum > youngest_inst) {
                break;
            }
            DPRINTF(LSQUnit, "Marking store as able to write back, PC "
                    "%s [sn:%lli]\n",
                    storeQueue[store_idx].inst->pcState(),
                    storeQueue[store_idx].inst->seqNum);

            storeQueue[store_idx].canWB = true;

            ++storesToWB;
        }

        incrStIdx(store_idx);
    }
}

template <class Impl>
void
LSQUnit<Impl>::writebackPendingStore()
{
    if (hasPendingPkt) {
        assert(pendingPkt != NULL);

        // If the cache is blocked, this will store the packet for retry.
        if (sendStore(pendingPkt)) {
            storePostSend(pendingPkt);
        }
        pendingPkt = NULL;
        hasPendingPkt = false;
    }
}

template <class Impl>
void
LSQUnit<Impl>::writebackStores()
{
    // First writeback the second packet from any split store that didn't
    // complete last cycle because there weren't enough cache ports available.
    if (TheISA::HasUnalignedMemAcc) {
        writebackPendingStore();
    }

    while (storesToWB > 0 &&
           storeWBIdx != storeTail &&
           storeQueue[storeWBIdx].inst &&
           storeQueue[storeWBIdx].canWB &&
           ((!needsTSO) || (!storeInFlight)) &&
           usedPorts < cachePorts) {

        if (isStoreBlocked) {
            DPRINTF(LSQUnit, "Unable to write back any more stores, cache"
                    " is blocked!\n");
            break;
        }

        // Store didn't write any data so no need to write it back to
        // memory.
        if (storeQueue[storeWBIdx].size == 0) {
            completeStore(storeWBIdx);

            incrStIdx(storeWBIdx);

            continue;
        }

        ++usedPorts;

        if (storeQueue[storeWBIdx].inst->isDataPrefetch()) {
            incrStIdx(storeWBIdx);

            continue;
        }

        assert(storeQueue[storeWBIdx].req);
        assert(!storeQueue[storeWBIdx].committed);

        if (TheISA::HasUnalignedMemAcc && storeQueue[storeWBIdx].isSplit) {
            assert(storeQueue[storeWBIdx].sreqLow);
            assert(storeQueue[storeWBIdx].sreqHigh);
        }

        DynInstPtr inst = storeQueue[storeWBIdx].inst;

        Request *req = storeQueue[storeWBIdx].req;
        RequestPtr sreqLow = storeQueue[storeWBIdx].sreqLow;
        RequestPtr sreqHigh = storeQueue[storeWBIdx].sreqHigh;

        storeQueue[storeWBIdx].committed = true;

        assert(!inst->memData);
        inst->memData = new uint8_t[req->getSize()];

        if (storeQueue[storeWBIdx].isAllZeros)
            memset(inst->memData, 0, req->getSize());
        else
            memcpy(inst->memData, storeQueue[storeWBIdx].data, req->getSize());

        PacketPtr data_pkt;
        PacketPtr snd_data_pkt = NULL;

        LSQSenderState *state = new LSQSenderState;
        state->isLoad = false;
        state->idx = storeWBIdx;
        state->inst = inst;

        if (!TheISA::HasUnalignedMemAcc || !storeQueue[storeWBIdx].isSplit) {

            // Build a single data packet if the store isn't split.
            data_pkt = Packet::createWrite(req);
            data_pkt->dataStatic(inst->memData);
            data_pkt->senderState = state;
        } else {
            // Create two packets if the store is split in two.
            data_pkt = Packet::createWrite(sreqLow);
            snd_data_pkt = Packet::createWrite(sreqHigh);

            data_pkt->dataStatic(inst->memData);
            snd_data_pkt->dataStatic(inst->memData + sreqLow->getSize());

            data_pkt->senderState = state;
            snd_data_pkt->senderState = state;

            state->isSplit = true;
            state->outstanding = 2;

            // Can delete the main request now.
            delete req;
            req = sreqLow;
        }

        DPRINTF(LSQUnit, "D-Cache: Writing back store idx:%i PC:%s "
                "to Addr:%#x, data:%#x [sn:%lli]\n",
                storeWBIdx, inst->pcState(),
                req->getPaddr(), (int)*(inst->memData),
                inst->seqNum);

        // @todo: Remove this SC hack once the memory system handles it.
        if (inst->isStoreConditional()) {
            assert(!storeQueue[storeWBIdx].isSplit);
            // Disable recording the result temporarily.  Writing to
            // misc regs normally updates the result, but this is not
            // the desired behavior when handling store conditionals.
            inst->recordResult(false);
            bool success = TheISA::handleLockedWrite(inst.get(), req, cacheBlockMask);
            inst->recordResult(true);

            if (!success) {
                // Instantly complete this store.
                DPRINTF(LSQUnit, "Store conditional [sn:%lli] failed.  "
                        "Instantly completing it.\n",
                        inst->seqNum);
                WritebackEvent *wb = new WritebackEvent(inst, data_pkt, this);
                cpu->schedule(wb, curTick() + 1);
                if (cpu->checker) {
                    // Make sure to set the LLSC data for verification
                    // if checker is loaded
                    inst->reqToVerify->setExtraData(0);
                    inst->completeAcc(data_pkt);
                }
                completeStore(storeWBIdx);
                incrStIdx(storeWBIdx);
                continue;
            }
        } else {
            // Non-store conditionals do not need a writeback.
            state->noWB = true;
        }

        bool split =
            TheISA::HasUnalignedMemAcc && storeQueue[storeWBIdx].isSplit;

        ThreadContext *thread = cpu->tcBase(lsqID);

        if (req->isMmappedIpr()) {
            assert(!inst->isStoreConditional());
            TheISA::handleIprWrite(thread, data_pkt);
            delete data_pkt;
            if (split) {
                assert(snd_data_pkt->req->isMmappedIpr());
                TheISA::handleIprWrite(thread, snd_data_pkt);
                delete snd_data_pkt;
                delete sreqLow;
                delete sreqHigh;
            }
            delete state;
            delete req;
            completeStore(storeWBIdx);
            incrStIdx(storeWBIdx);
        } else if (!sendStore(data_pkt)) {
            DPRINTF(IEW, "D-Cache became blocked when writing [sn:%lli], will"
                    "retry later\n",
                    inst->seqNum);

            // Need to store the second packet, if split.
            if (split) {
                state->pktToSend = true;
                state->pendingPacket = snd_data_pkt;
            }
        } else {

            // If split, try to send the second packet too
            if (split) {
                assert(snd_data_pkt);

                // Ensure there are enough ports to use.
                if (usedPorts < cachePorts) {
                    ++usedPorts;
                    if (sendStore(snd_data_pkt)) {
                        storePostSend(snd_data_pkt);
                    } else {
                        DPRINTF(IEW, "D-Cache became blocked when writing"
                                " [sn:%lli] second packet, will retry later\n",
                                inst->seqNum);
                    }
                } else {

                    // Store the packet for when there's free ports.
                    assert(pendingPkt == NULL);
                    pendingPkt = snd_data_pkt;
                    hasPendingPkt = true;
                }
            } else {

                // Not a split store.
                storePostSend(data_pkt);
            }
        }
    }

    // Not sure this should set it to 0.
    usedPorts = 0;

    assert(stores >= 0 && storesToWB >= 0);
}

/*template <class Impl>
void
LSQUnit<Impl>::removeMSHR(InstSeqNum seqNum)
{
    list<InstSeqNum>::iterator mshr_it = find(mshrSeqNums.begin(),
                                              mshrSeqNums.end(),
                                              seqNum);

    if (mshr_it != mshrSeqNums.end()) {
        mshrSeqNums.erase(mshr_it);
        DPRINTF(LSQUnit, "Removing MSHR. count = %i\n",mshrSeqNums.size());
    }
}*/

template <class Impl>
void
LSQUnit<Impl>::squash(const InstSeqNum &squashed_num)
{
    DPRINTF(LSQUnit, "Squashing until [sn:%lli]!"
            "(Loads:%i Stores:%i)\n", squashed_num, loads, stores);

    int load_idx = loadTail;
    decrLdIdx(load_idx);

    while (loads != 0 && loadQueue[load_idx]->seqNum > squashed_num) {
        DPRINTF(LSQUnit,"Load Instruction PC %s squashed, "
                "[sn:%lli]\n",
                loadQueue[load_idx]->pcState(),
                loadQueue[load_idx]->seqNum);

        if (isStalled() && load_idx == stallingLoadIdx) {
            stalled = false;
            stallingStoreIsn = 0;
            stallingLoadIdx = 0;
        }

        // Clear the smart pointer to make sure it is decremented.
        loadQueue[load_idx]->setSquashed();
        loadQueue[load_idx] = NULL;
        --loads;

        // Inefficient!
        loadTail = load_idx;

        decrLdIdx(load_idx);
        ++lsqSquashedLoads;
    }

    if (memDepViolator && squashed_num < memDepViolator->seqNum) {
        memDepViolator = NULL;
    }

    int store_idx = storeTail;
    decrStIdx(store_idx);

    while (stores != 0 &&
           storeQueue[store_idx].inst->seqNum > squashed_num) {
        // Instructions marked as can WB are already committed.
        if (storeQueue[store_idx].canWB) {
            break;
        }

        DPRINTF(LSQUnit,"Store Instruction PC %s squashed, "
                "idx:%i [sn:%lli]\n",
                storeQueue[store_idx].inst->pcState(),
                store_idx, storeQueue[store_idx].inst->seqNum);

        // I don't think this can happen.  It should have been cleared
        // by the stalling load.
        if (isStalled() &&
            storeQueue[store_idx].inst->seqNum == stallingStoreIsn) {
            panic("Is stalled should have been cleared by stalling load!\n");
            stalled = false;
            stallingStoreIsn = 0;
        }

        // Clear the smart pointer to make sure it is decremented.
        storeQueue[store_idx].inst->setSquashed();
        storeQueue[store_idx].inst = NULL;
        storeQueue[store_idx].canWB = 0;

        // Must delete request now that it wasn't handed off to
        // memory.  This is quite ugly.  @todo: Figure out the proper
        // place to really handle request deletes.
        delete storeQueue[store_idx].req;
        if (TheISA::HasUnalignedMemAcc && storeQueue[store_idx].isSplit) {
            delete storeQueue[store_idx].sreqLow;
            delete storeQueue[store_idx].sreqHigh;

            storeQueue[store_idx].sreqLow = NULL;
            storeQueue[store_idx].sreqHigh = NULL;
        }

        storeQueue[store_idx].req = NULL;
        --stores;

        // Inefficient!
        storeTail = store_idx;

        decrStIdx(store_idx);
        ++lsqSquashedStores;
    }
}

template <class Impl>
void
LSQUnit<Impl>::storePostSend(PacketPtr pkt)
{
    if (isStalled() &&
        storeQueue[storeWBIdx].inst->seqNum == stallingStoreIsn) {
        DPRINTF(LSQUnit, "Unstalling, stalling store [sn:%lli] "
                "load idx:%i\n",
                stallingStoreIsn, stallingLoadIdx);
        stalled = false;
        stallingStoreIsn = 0;
        iewStage->replayMemInst(loadQueue[stallingLoadIdx]);
    }

    if (!storeQueue[storeWBIdx].inst->isStoreConditional()) {
        // The store is basically completed at this time. This
        // only works so long as the checker doesn't try to
        // verify the value in memory for stores.
        storeQueue[storeWBIdx].inst->setCompleted();

        if (cpu->checker) {
            cpu->checker->verify(storeQueue[storeWBIdx].inst);
        }
    }

    if (needsTSO) {
        storeInFlight = true;
    }

    incrStIdx(storeWBIdx);
}

template <class Impl>
void
LSQUnit<Impl>::writeback(DynInstPtr &inst, PacketPtr pkt)
{
    iewStage->wakeCPU();

    // Squashed instructions do not need to complete their access.
    if (inst->isSquashed()) {
        assert(!inst->isStore());
        ++lsqIgnoredResponses;
        return;
    }

    if (!inst->isExecuted()) {
        inst->setExecuted();

        if (inst->fault == NoFault) {
            // Complete access to copy data to proper place.
            inst->completeAcc(pkt);
        } else {
            // If the instruction has an outstanding fault, we cannot complete
            // the access as this discards the current fault.

            // If we have an outstanding fault, the fault should only be of
            // type ReExec.
            assert(dynamic_cast<ReExec*>(inst->fault.get()) != nullptr);

            DPRINTF(LSQUnit, "Not completing instruction [sn:%lli] access "
                    "due to pending fault.\n", inst->seqNum);
        }
    }

    // Need to insert instruction into queue to commit
    iewStage->instToCommit(inst);

    iewStage->activityThisCycle();

    // see if this load changed the PC
    iewStage->checkMisprediction(inst);
}

template <class Impl>
void
LSQUnit<Impl>::completeStore(int store_idx)
{
    assert(storeQueue[store_idx].inst);
    storeQueue[store_idx].completed = true;
    --storesToWB;
    // A bit conservative because a store completion may not free up entries,
    // but hopefully avoids two store completions in one cycle from making
    // the CPU tick twice.
    cpu->wakeCPU();
    cpu->activityThisCycle();

    if (store_idx == storeHead) {
        do {
            incrStIdx(storeHead);

            --stores;
        } while (storeQueue[storeHead].completed &&
                 storeHead != storeTail);

        iewStage->updateLSQNextCycle = true;
    }

    DPRINTF(LSQUnit, "Completing store [sn:%lli], idx:%i, store head "
            "idx:%i\n",
            storeQueue[store_idx].inst->seqNum, store_idx, storeHead);

#if TRACING_ON
    if (DTRACE(O3PipeView)) {
        storeQueue[store_idx].inst->storeTick =
            curTick() - storeQueue[store_idx].inst->fetchTick;
    }
#endif

    if (isStalled() &&
        storeQueue[store_idx].inst->seqNum == stallingStoreIsn) {
        DPRINTF(LSQUnit, "Unstalling, stalling store [sn:%lli] "
                "load idx:%i\n",
                stallingStoreIsn, stallingLoadIdx);
        stalled = false;
        stallingStoreIsn = 0;
        iewStage->replayMemInst(loadQueue[stallingLoadIdx]);
    }

    storeQueue[store_idx].inst->setCompleted();

    if (needsTSO) {
        storeInFlight = false;
    }

    // Tell the checker we've completed this instruction.  Some stores
    // may get reported twice to the checker, but the checker can
    // handle that case.
    if (cpu->checker) {
        cpu->checker->verify(storeQueue[store_idx].inst);
    }
}

template <class Impl>
bool
LSQUnit<Impl>::sendStore(PacketPtr data_pkt)
{
    if (!dcachePort->sendTimingReq(data_pkt)) {
        // Need to handle becoming blocked on a store.
        isStoreBlocked = true;
        ++lsqCacheBlocked;
        assert(retryPkt == NULL);
        retryPkt = data_pkt;
        return false;
    }
    return true;
}

template <class Impl>
void
LSQUnit<Impl>::recvRetry()
{
    if (isStoreBlocked) {
        DPRINTF(LSQUnit, "Receiving retry: store blocked\n");
        assert(retryPkt != NULL);

        LSQSenderState *state =
            dynamic_cast<LSQSenderState *>(retryPkt->senderState);

        if (dcachePort->sendTimingReq(retryPkt)) {
            // Don't finish the store unless this is the last packet.
            if (!TheISA::HasUnalignedMemAcc || !state->pktToSend ||
                    state->pendingPacket == retryPkt) {
                state->pktToSend = false;
                storePostSend(retryPkt);
            }
            retryPkt = NULL;
            isStoreBlocked = false;

            // Send any outstanding packet.
            if (TheISA::HasUnalignedMemAcc && state->pktToSend) {
                assert(state->pendingPacket);
                if (sendStore(state->pendingPacket)) {
                    storePostSend(state->pendingPacket);
                }
            }
        } else {
            // Still blocked!
            ++lsqCacheBlocked;
        }
    }
}

template <class Impl>
inline void
LSQUnit<Impl>::incrStIdx(int &store_idx) const
{
    if (++store_idx >= SQEntries)
        store_idx = 0;
}

template <class Impl>
inline void
LSQUnit<Impl>::decrStIdx(int &store_idx) const
{
    if (--store_idx < 0)
        store_idx += SQEntries;
}

template <class Impl>
inline void
LSQUnit<Impl>::incrLdIdx(int &load_idx) const
{
    if (++load_idx >= LQEntries)
        load_idx = 0;
}

template <class Impl>
inline void
LSQUnit<Impl>::decrLdIdx(int &load_idx) const
{
    if (--load_idx < 0)
        load_idx += LQEntries;
}

template <class Impl>
void
LSQUnit<Impl>::dumpInsts() const
{
    cprintf("Load store queue: Dumping instructions.\n");
    cprintf("Load queue size: %i\n", loads);
    cprintf("Load queue: ");

    int load_idx = loadHead;

    while (load_idx != loadTail && loadQueue[load_idx]) {
        const DynInstPtr &inst(loadQueue[load_idx]);
        cprintf("%s.[sn:%i] ", inst->pcState(), inst->seqNum);

        incrLdIdx(load_idx);
    }
    cprintf("\n");

    cprintf("Store queue size: %i\n", stores);
    cprintf("Store queue: ");

    int store_idx = storeHead;

    while (store_idx != storeTail && storeQueue[store_idx].inst) {
        const DynInstPtr &inst(storeQueue[store_idx].inst);
        cprintf("%s.[sn:%i] ", inst->pcState(), inst->seqNum);

        incrStIdx(store_idx);
    }

    cprintf("\n");
}

#endif//__CPU_O3_LSQ_UNIT_IMPL_HH__


text:
This directory contains all compatible WebRTC Experiments.

----

#### License

[WebRTC Experiments](https://github.com/muaz-khan/WebRTC-Experiment) are released under [MIT licence](https://webrtc-experiment.appspot.com/licence/) . Copyright (c) 2013 [Muaz Khan](https://plus.google.com/100325991024054712503).


text:
from sys import argv
from jsonschema import validate, ValidationError
import json
import os

script, config_path = argv

def validate_config(config_path):
    config_schema_path = os.path.join(
       os.path.dirname(os.path.abspath(script)),
       'schemas/config.schema')
    try:
        config_schema_json = json.load(open(config_schema_path))
        config_json = json.load(open(config_path))
        validate(config_json, config_schema_json)
        print("Configuration file sucessfully validated.")
        return config_json
    except ValueError as ve:
        print(ve)
        #print(f"Invalid configuration file {os.path.abspath(config_path)}")

config = validate_config(config_path)

verbose = config['verbose']

def conlog(s):
    if verbose:
        print(s)

root_config = config['root']
comp_config = config['comp']

src_path = os.path.abspath(config['src_path'])

def find_rdx_paths(path):
    rdx_paths = []
    for root, dirs, files in os.walk(path):
        for _file in files:
                if _file.endswith(".rdx"):
                    file_path = os.path.abspath(os.path.join(root,_file))
                    rdx_paths.append(file_path)
    return rdx_paths

def find_root_component(rdx_paths):
    for rdx_path in rdx_paths:
        rdx_f = open(rdx_path)
        rdx_json = json.load(rdx_f)
        if(rdx_json['root'] == True):
            try:
                new_paths = [c_path for c_path in rdx_paths if c_path != rdx_path]
                return rdx_path, new_paths
            except ValueError as ve:
                print (ve)

def validate_rdx(rdx_path, schema_path):
    schema_f = open(schema_path)
    rdx_f = open(rdx_path)
    schema_json = json.load(schema_f)
    rdx_json = json.load(rdx_f)
    try:
        print(f"Validating rdx: {os.path.basename(rdx_path)}")
        validate(rdx_json, schema_json)
        #Source dir is needed for injecting actions
        rdx_json['src_dir'] = os.path.abspath(os.path.dirname(rdx_path))
        return rdx_json
    except ValidationError as ve:
        print(f"Error validating {ve}")
        print(f"Make sure {rdx_path} is a valid schema!")

def validate_root_rdx(root_path):
    root_schema_path = os.path.abspath(root_config['schema_path'])
    schema_path = os.path.abspath(root_schema_path)
    return validate_rdx(root_path, schema_path)

    root_json = json.load(open(root_path))

def validate_comp_rdx(comp_paths):
    rest_valid_json = []
    comp_schema_path = os.path.abspath(comp_config['schema_path'])
    for rdx_path in comp_paths:
        print (f"Validating component rdx: {os.path.basename(rdx_path)}")
        rdx_json = validate_rdx(rdx_path, comp_schema_path)
        rest_valid_json.append(rdx_json)

    return rest_valid_json

def validate_all_rdx(src_path):
    rdx_paths = find_rdx_paths(src_path)
    root_path, rest_paths = find_root_component(rdx_paths)
    root_valid_json = validate_root_rdx(root_path)
    comp_valid_json = validate_comp_rdx(rest_paths)
    return (root_valid_json, comp_valid_json)

root_valid_rdx, comp_valid_rdx = validate_all_rdx(src_path)

def get_payload_types(comp_rdx):
    payloads = []
    for idx, rdx in enumerate(comp_rdx):
        payloads.append(rdx['name']+'PayloadTypes')

    return payloads

def get_payload_types_paths(comp_rdx):
    payload_paths = []
    for rdx in comp_rdx:
        payload_paths.append(rdx['src_dir'])

    return payload_paths

def get_action_types(comp_list):
    action_types = []
    for comp in comp_list:
        action_types.append(comp['name']+'ActionTypes')
    return action_types

def concat_payload_paths(payload_path_list):
    concat_paths = ''
    for idx, path in enumerate(payload_path_list):
        concat_paths += f'import {path}/Actions\n'

    return concat_paths

def union_types(types_list):
    concat_types = ''
    for idx, typestr in enumerate(types_list):
        if idx > 0 and idx < len(types_list):
            concat_types += ' | '

        concat_types += typestr

    return concat_types

def get_action_types(comp_list):
    action_types = []
    for comp in comp_list:
        action_types.append(comp['name']+'ActionTypes')
    return action_types

def make_root_imports(comp_rdx):
    import_actions = []
    import_tuple = []
    for comp in comp_rdx:
        t_string = comp['name']+'PayloadTypes, '+comp['name']+'Actions'
        import_tuple.append( (t_string, comp['src_dir']) )
        import_actions.append(t_string)
    print(f'import_actions: {import_actions}')
    print(f'import_paths: {import_tuple}')
    return import_tuple

            #('#IMPORT_OTHER_ACTIONS.PATHS#', import_paths)]
def write_root_imports(template_str, import_tuple):
    temp_list = template_str.split('\n')
    new_str = ''
    for line in temp_list:
        new_line = line
        if "#IMPORT_OTHER_ACTIONS#" in line:
            print("found import!")
            for (types, path) in import_tuple:
                new_line = new_line.replace("#IMPORT_OTHER_ACTIONS#", types)
                new_line = new_line.replace("#IMPORT_OTHER_ACTIONS.PATHS#", path)
                new_str += new_line+'\n'
                new_line = line
            line = '\n'   #delete line with pattern on it once we are done iterating through
                        # all the types and paths
        new_str += line+'\n'
    return new_str

shared_keys = dict([
    ("#ROOT.NAME#", root_valid_rdx['name']),
    ("#ROOT.PATH#", root_valid_rdx['src_dir'])
]).items()

root_keys = dict([
    ("#STATE_CLASS#", root_valid_rdx['initialState']['class']),
    ("#STATE_PATH#", root_valid_rdx['initialState']['path']),
    ("#ROOT.ACTION_TYPES#", root_valid_rdx['actionTypes']),
    ("#OTHER_ACTIONS#", union_types(get_action_types(comp_valid_rdx))),
    ("#OTHER_PAYLOAD_TYPES#", union_types(get_payload_types(comp_valid_rdx))),
    ("#OTHER_PAYLOAD_TYPES.PATHS#", concat_payload_paths(get_payload_types_paths(comp_valid_rdx)))
    ]).items()

def make_comp_keys(comp_rdx):
    return dict([
        ("#COMPONENT.NAME#", comp_rdx['name']),
        ("#COMPONENT.ACTION_TYPES#", comp_rdx['actionTypes']),
        ("#COMPONENT.PAYLOAD_TYPES#", union_types(comp_rdx['payloadTypes']))
    ]).items()

def write_from_keys(rdx_json, template_str, keys):
    comp_str = ''
    for line in template_str:
        for key,value in keys:
            if isinstance(value, list):
                dupline = line
                new_line = ""
                for _value in value:
                    if key in dupline:
                        new_line += dupline.replace(key, _value)
                        line = new_line
            else:
                line = line.replace(key, value)
        comp_str += line
    return comp_str

def write_shared_values(rdx_json, template_path):
    new_data = []
    with open(template_path) as action_template:
        for line in action_template:

            for key,value in shared_keys:
                if key in line:
                    line = line.replace(key, value)
            new_data.append(line)
    return new_data

def make_comp_actions(rdx_json, template_path):
    comp_strs = []
    for rdx in rdx_json:
        template_str = write_shared_values(rdx, template_path)
        comp_strs.append({  'path':rdx['src_dir'],
                            'file_name':rdx['name'],
                            'value':write_from_keys(rdx, template_str, make_comp_keys(rdx))})
    return comp_strs

def make_root_actions(root_json, template_path):
    root_actions = {}
    template_str = write_shared_values(root_json, template_path)
    template_str = write_from_keys(root_json, template_str, root_keys)
    template_str = write_root_imports(template_str, make_root_imports(comp_valid_rdx))

    return template_str

comp_strs = make_comp_actions(comp_valid_rdx, comp_config['action_path'])
root_strs = make_root_actions(root_valid_rdx, root_config['action_path'])

print(root_strs)


text:
package skinny.validator

import scala.language.reflectiveCalls
import skinny.util.DateTimeUtil

/**
 * Built-in validation rules.
 */

// ----
// param("x" -> "") is notNull

object notNull extends ValidationRule {
  def name = "notNull"
  def isValid(v: Any) = v != null
}

// ----
// param("x" -> "y") is required

object required extends required(true)

case class required(trim: Boolean = true) extends ValidationRule {
  def name = "required"
  def isValid(v: Any) = !isEmpty(v) && {
    if (trim) v.toString.trim.length > 0
    else v.toString.length > 0
  }
}

// ----
// param("x" -> "y") is notEmpty
// param("list" -> Seq(1,2,3)) is notEmpty

object notEmpty extends notEmpty(true)

case class notEmpty(trim: Boolean = true) extends ValidationRule {
  def name = "notEmpty"
  def isValid(v: Any) = !isEmpty(v) && {
    toHasSize(v).map(x => x.size > 0).getOrElse {
      if (trim) v.toString.trim.length > 0
      else v.toString.length > 0
    }
  }
}

case class length(len: Int) extends ValidationRule {
  def name = "length"
  override def messageParams = Seq(len.toString)
  def isValid(v: Any) = isEmpty(v) || {
    toHasSize(v).map(x => x.size == len)
      .getOrElse(v.toString.length == len)
  }
}

// ----
// param("x" -> "yyyymmdd") is minLength(3)
// param("list" -> (1 to 5)) is minLength(3)

case class minLength(min: Int) extends ValidationRule {
  def name = "minLength"
  override def messageParams = Seq(min.toString)
  def isValid(v: Any) = isEmpty(v) || {
    toHasSize(v).map(x => x.size >= min)
      .getOrElse(v.toString.length >= min)
  }
}

// ----
// param("x" -> "y") is maxLength(3)
// param("list" -> Seq(1,2)) is maxLength(3)

case class maxLength(max: Int) extends ValidationRule {
  def name = "maxLength"
  override def messageParams = Seq(max.toString)
  def isValid(v: Any) = isEmpty(v) || {
    toHasSize(v).map(x => x.size <= max)
      .getOrElse(v.toString.length <= max)
  }
}

// ----
// param("x" -> "y") is minMaxLength(3, 6)
// param("list" -> Seq(1,2,3,4)) is minMaxLength(3, 6)

case class minMaxLength(min: Int, max: Int) extends ValidationRule {
  def name = "minMaxLength"
  override def messageParams = Seq(min.toString, max.toString)
  def isValid(v: Any) = isEmpty(v) || {
    toHasSize(v).map(x => x.size >= min && x.size <= max).getOrElse {
      v.toString.length >= min && v.toString.length <= max
    }
  }
}

// ----
// param("x" -> "123") is numeric
// param("x" -> 0.123D) is numeric

object numeric extends ValidationRule {
  def name = "numeric"
  def isValid(v: Any) = isEmpty(v) || isNumeric(v)
}

// ----
// param("x" -> "123") is intValue
object intValue extends ValidationRule {
  def name = "intValue"
  def isValid(v: Any) = isEmpty(v) || isInt(v)
}

// ----
// param("x" -> "123") is longValue

object longValue extends ValidationRule {
  def name = "longValue"
  def isValid(v: Any) = isEmpty(v) || isLong(v)
}

// ----
// param("x" -> "1.7976931348623157E308") is doubleValue

object doubleValue extends ValidationRule {
  def name = "doubleValue"
  def isValid(v: Any) = isEmpty(v) || isDouble(v)
}

// ----
// param("x" -> "3.4028235E38") is floatValue

object floatValue extends ValidationRule {
  def name = "floatValue"
  def isValid(v: Any) = isEmpty(v) || isFloat(v)
}

// ----
// param("x" -> 4) is intMinMaxValue(3, 5)

case class intMinMaxValue(min: Int, max: Int) extends ValidationRule {
  def name = "intMinMaxValue"
  override def messageParams = Seq(min.toString, max.toString)
  def isValid(v: Any) = isEmpty(v) || (isInt(v) && toInt(v) >= min && toInt(v) <= max)
}

// ----
// param("x" -> 2) is intMinValue(3)

case class intMinValue(min: Int) extends ValidationRule {
  def name = "intMinValue"
  override def messageParams = Seq(min.toString)
  def isValid(v: Any) = isEmpty(v) || (isInt(v) && toInt(v) >= min)
}

// ----
// param("x" -> 4) is intMaxValue(5)

case class intMaxValue(max: Int) extends ValidationRule {
  def name = "intMaxValue"
  override def messageParams = Seq(max.toString)
  def isValid(v: Any) = isEmpty(v) || (isInt(v) && toInt(v) <= max)
}

// ----
// param("x" -> "3") is longMinMaxValue(3L, 5L)

case class longMinMaxValue(min: Long, max: Long) extends ValidationRule {
  def name = "longMinMaxValue"
  override def messageParams = Seq(min.toString, max.toString)
  def isValid(v: Any) = isEmpty(v) || (isLong(v) && toLong(v) >= min && toLong(v) <= max)
}

// ----
// param("x" -> 5) is longMinValue(3L)

case class longMinValue(min: Long) extends ValidationRule {
  def name = "longMinValue"
  override def messageParams = Seq(min.toString)
  def isValid(v: Any) = isEmpty(v) || (isLong(v) && toLong(v) >= min)
}

// ----
// param("x" -> 1.0D) is longMaxValue(5L)

case class longMaxValue(max: Long) extends ValidationRule {
  def name = "longMaxValue"
  override def messageParams = Seq(max.toString)
  def isValid(v: Any) = isEmpty(v) || (isLong(v) && toLong(v) <= max)
}

// ----
// param("x" -> "4D") is doubleMinMaxValue(3D, 5D)

case class doubleMinMaxValue(min: Double, max: Double) extends ValidationRule {
  def name = "doubleMinMaxValue"
  override def messageParams = Seq(min.toString, max.toString)
  def isValid(v: Any) = isEmpty(v) || (isDouble(v) && toDouble(v) >= min && toDouble(v) <= max)
}

// ----
// param("x" -> 2D) is doubleMinValue(3D)

case class doubleMinValue(min: Double) extends ValidationRule {
  def name = "doubleMinValue"
  override def messageParams = Seq(min.toString)
  def isValid(v: Any) = isEmpty(v) || (isDouble(v) && toDouble(v) >= min)
}

// ----
// param("x" -> 6D) is doubleMaxValue(5D)

case class doubleMaxValue(max: Double) extends ValidationRule {
  def name = "doubleMaxValue"
  override def messageParams = Seq(max.toString)
  def isValid(v: Any) = isEmpty(v) || (isDouble(v) && toDouble(v) <= max)
}

// ----
// param("x" -> "4F") is floatMinMaxValue(3F, 5F)

case class floatMinMaxValue(min: Float, max: Float) extends ValidationRule {
  def name = "floatMinMaxValue"
  override def messageParams = Seq(min.toString, max.toString)
  def isValid(v: Any) = isEmpty(v) || (isFloat(v) && toFloat(v) >= min && toFloat(v) <= max)
}

// ----
// param("x" -> 2F) is floatMinValue(3F)

case class floatMinValue(min: Float) extends ValidationRule {
  def name = "floatMinValue"
  override def messageParams = Seq(min.toString)
  def isValid(v: Any) = isEmpty(v) || (isFloat(v) && toFloat(v) >= min)
}

// ----
// param("x" -> 6F) is floatMaxValue(5F)

case class floatMaxValue(max: Float) extends ValidationRule {
  def name = "floatMaxValue"
  override def messageParams = Seq(max.toString)
  def isValid(v: Any) = isEmpty(v) || (isFloat(v) && toFloat(v) <= max)
}

// ----
// param("pair" -> ("pass", "pass")) are same

object same extends ValidationRule {
  def name = "same"
  def isValid(pair: Any) = {
    val (a, b) = pair.asInstanceOf[(Any, Any)]
    if (a.isInstanceOf[Option[_]] && b.isInstanceOf[Option[_]]) {
      val (x, y) = pair.asInstanceOf[(Option[Any], Option[Any])]
      (x.isEmpty && y.isEmpty) || (x.isDefined && y.isDefined && x.get == y.get)
    } else {
      a == b
    }
  }
}

// ----
// param("email" -> "alice@example.com") is email
// [NOTE] This is not a complete solution

object email extends ValidationRule {
  def name = "email"
  def isValid(v: Any) = isEmpty(v) ||
    """\A([^@\s]+)@((?:[-a-z0-9]+\.)+[a-z]{2,})\z""".r.findFirstIn(v.toString).isDefined
}

// ----
// param("time" -> new java.util.Date(123L)) is past
// param("time" -> org.joda.time.DateTime.now.minusDays(3)) is past

object past extends ValidationRule {
  def name = "past"
  def isValid(v: Any): Boolean = {
    if (v != null) {
      toHasGetTime(v) match {
        case Some(time) => time.getTime < nowMillis()
        case _ => false
      }
    } else false
  }
}

// ----
// param("time" -> new java.util.Date) is future

object future extends ValidationRule {
  def name = "future"
  def isValid(v: Any): Boolean = {
    if (v != null) {
      toHasGetTime(v) match {
        case Some(time) => time.getTime > nowMillis()
        case _ => false
      }
    } else false
  }
}

// ----
// param("createdAt" -> "2013-01-02 03:04:05") is dateTimeFormat
object dateTimeFormat extends ValidationRule {
  def name = "dateTimeFormat"
  def isValid(v: Any): Boolean = isEmpty(v) || {
    try DateTimeUtil.parseDateTime(v.toString) != null
    catch { case scala.util.control.NonFatal(e) => false }
  }
}

// ----
// param("birthday" -> "2011-06-22") is dateFormat
object dateFormat extends ValidationRule {
  def name = "dateFormat"
  def isValid(v: Any): Boolean = isEmpty(v) || {
    try DateTimeUtil.parseLocalDate(v.toString) != null
    catch { case scala.util.control.NonFatal(e) => false }
  }
}

// ----
// param("timeToWakeUp" -> "12:34:56") is timeFormat
object timeFormat extends ValidationRule {
  def name = "timeFormat"
  def isValid(v: Any): Boolean = isEmpty(v) || {
    try DateTimeUtil.parseLocalTime(v.toString) != null
    catch { case scala.util.control.NonFatal(e) => false }
  }
}


text:
const ExtractTextPlugin = require('extract-text-webpack-plugin');
const path = require('path');

const extractSass = new ExtractTextPlugin({
    filename: "style.css",
    disable: process.env.NODE_ENV === "development"
});

const config = {

    entry: path.resolve(__dirname, 'src/main.js'),
    output: {
        path: path.resolve(__dirname, 'wwwroot'),
        filename: 'bundle.js'
    },
    module: {
        rules: [
            {
                test: /\.jsx?$/,
                exclude: '/node_modules',
                loader: 'babel-loader',
                query: {
                    presets: ['es2015', 'react']
                }
            },
            {
                test: /\.json$/,
                loader: 'json-loader'
            },
            {
                test: /\.css$/,
                use: ExtractTextPlugin.extract({
                    use: 'css-loader'
                })
            },
            {
                test: /(\.scss$|\.sass$)/,
                use: extractSass.extract({
                    use: [
                        { loader: 'css-loader' },
                        { loader: 'sass-loader' }
                    ],
                    fallback: 'style-loader'
                })
            }
        ]
    },
    plugins: [
        extractSass
    ],
    devServer: {
        contentBase: path.join(__dirname, 'wwwroot'),
        compress: true,
        port: 5000
    }

}

module.exports = config;


text:
package org.axonframework.spring.config.annotation;

import org.axonframework.commandhandling.AnnotationCommandHandlerAdapter;
import org.axonframework.commandhandling.CommandHandler;
import org.axonframework.commandhandling.CommandMessage;
import org.axonframework.commandhandling.CommandMessageHandler;
import org.axonframework.common.annotation.AnnotationUtils;
import org.axonframework.messaging.MessageHandler;
import org.axonframework.messaging.annotation.HandlerDefinition;
import org.axonframework.messaging.annotation.ParameterResolverFactory;
import org.axonframework.spring.config.AbstractAnnotationHandlerBeanPostProcessor;
import org.springframework.util.ReflectionUtils;

import java.lang.reflect.Method;
import java.util.concurrent.atomic.AtomicBoolean;

/**
 * Spring Bean post processor that automatically generates an adapter for each bean containing {@link CommandHandler}
 * annotated methods.
 *
 * @author Allard Buijze
 * @since 0.5
 */
public class AnnotationCommandHandlerBeanPostProcessor
        extends AbstractAnnotationHandlerBeanPostProcessor<MessageHandler<CommandMessage<?>>, AnnotationCommandHandlerAdapter<?>> {

    @Override
    protected Class<?>[] getAdapterInterfaces() {
        return new Class[]{CommandMessageHandler.class};
    }

    @Override
    protected boolean isPostProcessingCandidate(Class<?> targetClass) {
        return hasCommandHandlerMethod(targetClass);
    }

    @Override
    protected AnnotationCommandHandlerAdapter<?> initializeAdapterFor(Object bean,
                                                                   ParameterResolverFactory parameterResolverFactory,
                                                                   HandlerDefinition handlerDefinition) {
        return new AnnotationCommandHandlerAdapter<>(bean, parameterResolverFactory, handlerDefinition);
    }

    private boolean hasCommandHandlerMethod(Class<?> beanClass) {
        final AtomicBoolean result = new AtomicBoolean(false);
        ReflectionUtils.doWithMethods(beanClass, new HasCommandHandlerAnnotationMethodCallback(result));
        return result.get();
    }

    private static final class HasCommandHandlerAnnotationMethodCallback implements ReflectionUtils.MethodCallback {

        private final AtomicBoolean result;

        private HasCommandHandlerAnnotationMethodCallback(AtomicBoolean result) {
            this.result = result;
        }

        /**
         * {@inheritDoc}
         */
        @Override
        public void doWith(Method method) throws IllegalArgumentException {
            if (AnnotationUtils.findAnnotationAttributes(method, CommandHandler.class).isPresent()) {
                result.set(true);
            }
        }
    }
}


text:
package cmd

import (
	"os"
	"path/filepath"
	"runtime"

	"github.com/spf13/cobra"

	"github.com/Shopify/themekit/src/cmdutil"
	"github.com/Shopify/themekit/src/colors"
	"github.com/Shopify/themekit/src/env"
	"github.com/Shopify/themekit/src/release"
	"github.com/Shopify/themekit/src/util"
)

const afterUpdateMessage = `Successfully updated to theme kit version %v, for more information on this release please see the change log
	https://github.com/Shopify/themekit/blob/master/changelog.txt

If you have troubles with this release please report them to
	https://github.com/Shopify/themekit/issues

If your troubles are preventing you from working you can roll back to the previous version using the command
	'theme update --version=v%s'
 `

var (
	flags cmdutil.Flags

	// ThemeCmd is the main entry point to the theme kit command line interface.
	ThemeCmd = &cobra.Command{
		Use:   "theme",
		Short: "Theme Kit is a tool kit for manipulating shopify themes",
		Long: `Theme Kit is a tool kit for manipulating shopify themes

Theme Kit is a fast and cross platform tool that enables you to build shopify themes with ease.

Complete documentation is available at https://shopify.dev/tools/theme-kit.`,
		SilenceUsage:  true,
		SilenceErrors: true,
		PersistentPreRun: func(cmd *cobra.Command, args []string) {
			if !flags.DisableUpdateNotifier && release.IsUpdateAvailable() {
				colors.ColorStdOut.Print(colors.Yellow("An update for Themekit is available. To update please run `theme update`"))
			}
		},
		PersistentPostRun: func(cmd *cobra.Command, args []string) {
			// env validation requires a theme id. setting a dummy one here if not provided
			if flags.ThemeID == "" {
				flags.ThemeID = "1337"
			}
			cmdutil.ForDefaultClient(flags, args, func(ctx *cmdutil.Ctx) error {
				if !flags.DisableThemeKitAccessNotifier && !util.IsThemeAccessPassword(ctx.Env.Password) {
					colors.ColorStdOut.Print(colors.Yellow("* Build themes without private apps. Learn more about the Theme Access app: https://shopify.dev/themes/tools/theme-access"))
				}
				return nil
			})
		},
	}

	updateCmd = &cobra.Command{
		Use:   "update",
		Short: "Update Theme kit to the newest version.",
		Long: `Update will check for a new release, then if there is an applicable update it will download it and apply it.

 For more information, refer to https://shopify.dev/tools/theme-kit/troubleshooting.
 `,
		RunE: func(cmd *cobra.Command, args []string) (err error) {
			colors.ColorStdOut.Printf("Updating from %s to %s", colors.Yellow(release.ThemeKitVersion), colors.Yellow(flags.Version))
			if err = release.Install(flags.Version); err == nil {
				colors.ColorStdOut.Printf(afterUpdateMessage, colors.Green(flags.Version), colors.Yellow(release.ThemeKitVersion))
			}
			return
		},
	}

	versionCmd = &cobra.Command{
		Use:   "version",
		Short: "Print the version number of Theme Kit",
		Long:  `All software has versions. This is Theme Kit's version.`,
		Run: func(cmd *cobra.Command, args []string) {
			colors.ColorStdOut.Printf("ThemeKit %s %s/%s", release.ThemeKitVersion.String(), runtime.GOOS, runtime.GOARCH)
		},
	}
)

func init() {
	pwd, _ := os.Getwd()
	defaultConfigPath := filepath.Join(pwd, "config.yml")

	ThemeCmd.PersistentFlags().StringVarP(&flags.ConfigPath, "config", "c", defaultConfigPath, "path to config.yml")
	ThemeCmd.PersistentFlags().StringVar(&flags.VariableFilePath, "vars", "", "path to an file that defines environment variables")
	ThemeCmd.PersistentFlags().StringArrayVarP(&flags.Environments, "env", "e", []string{env.Default.Name}, "environment to run the command")
	ThemeCmd.PersistentFlags().StringVarP(&flags.Directory, "dir", "d", "", "directory that command will take effect. (default current directory)")
	ThemeCmd.PersistentFlags().StringVarP(&flags.Password, "password", "p", "", "theme password. This will override what is in your config.yml")
	ThemeCmd.PersistentFlags().StringVarP(&flags.ThemeID, "themeid", "t", "", "theme id. This will override what is in your config.yml")
	ThemeCmd.PersistentFlags().StringVarP(&flags.Domain, "store", "s", "", "your shopify domain. This will override what is in your config.yml")
	ThemeCmd.PersistentFlags().StringVar(&flags.Proxy, "proxy", "", "proxy for all theme requests. This will override what is in your config.yml")
	ThemeCmd.PersistentFlags().DurationVar(&flags.Timeout, "timeout", 0, "the timeout to kill any stalled processes. This will override what is in your config.yml")
	ThemeCmd.PersistentFlags().BoolVarP(&flags.Verbose, "verbose", "v", false, "Enable more verbose output from the running command.")
	ThemeCmd.PersistentFlags().BoolVarP(&flags.DisableUpdateNotifier, "no-update-notifier", "", false, "Stop theme kit from notifying about updates.")
	ThemeCmd.PersistentFlags().StringArrayVar(&flags.IgnoredFiles, "ignored-file", []string{}, "A single file to ignore, use the flag multiple times to add multiple.")
	ThemeCmd.PersistentFlags().StringArrayVar(&flags.Ignores, "ignores", []string{}, "A path to a file that contains ignore patterns.")
	ThemeCmd.PersistentFlags().BoolVar(&flags.DisableIgnore, "no-ignore", false, "Will disable config ignores so that all files can be changed")
	ThemeCmd.PersistentFlags().BoolVar(&flags.AllowLive, "allow-live", false, "Will allow themekit to make changes to the live theme on the store.")
	ThemeCmd.PersistentFlags().BoolVarP(&flags.DisableThemeKitAccessNotifier, "no-theme-kit-access-notifier", "", false, "Stop theme kit from notifying about Theme Access.")

	watchCmd.Flags().StringVarP(&flags.Notify, "notify", "n", "", "file to touch or url to notify when a file has been changed")
	watchCmd.Flags().BoolVarP(&flags.AllEnvs, "allenvs", "a", false, "run command with all environments")
	removeCmd.Flags().BoolVarP(&flags.AllEnvs, "allenvs", "a", false, "run command with all environments")
	openCmd.Flags().BoolVarP(&flags.AllEnvs, "allenvs", "a", false, "run command with all environments")
	downloadCmd.Flags().BoolVarP(&flags.AllEnvs, "allenvs", "a", false, "run command with all environments")
	deployCmd.Flags().BoolVarP(&flags.AllEnvs, "allenvs", "a", false, "run command with all environments")
	updateCmd.Flags().StringVar(&flags.Version, "version", "latest", "version of themekit to install")
	newCmd.Flags().StringVarP(&flags.Name, "name", "n", "", "a name to define your theme on your shopify admin")
	openCmd.Flags().BoolVarP(&flags.Edit, "edit", "E", false, "open the web editor for the theme.")
	openCmd.Flags().StringVarP(&flags.With, "browser", "b", "", "name of the browser to open the url. the name should match the name of browser on your system.")
	getCmd.Flags().BoolVarP(&flags.List, "list", "l", false, "list available themes.")
	deployCmd.Flags().BoolVarP(&flags.NoDelete, "nodelete", "n", false, "do not delete files on shopify during deploy.")
	openCmd.Flags().BoolVar(&flags.HidePreviewBar, "hidepb", false, "run command with all environments")

	getCmd.Flags().BoolVar(&flags.Live, "live", false, "will allow themekit to autofill the theme ID as the currently published theme ID")
	downloadCmd.Flags().BoolVar(&flags.Live, "live", false, "will allow themekit to autofill the theme ID as the currently published theme ID")
	configureCmd.Flags().BoolVar(&flags.Live, "live", false, "will allow themekit to autofill the theme ID as the currently published theme ID")

	ThemeCmd.AddCommand(
		configureCmd,
		deployCmd,
		downloadCmd,
		getCmd,
		newCmd,
		openCmd,
		publishCmd,
		removeCmd,
		updateCmd,
		versionCmd,
		watchCmd,
	)
}


text:
export { default as routerBase } from './router-base.js';
export { default as siteHead } from './site-head.js';
export { default as siteNavigation } from './site-navigation.js';