text:
require_relative 'db_queries'
require_dependency 'carto/db/database'
require_dependency 'carto/db/user_schema_mover'
require 'cartodb/sequel_connection_helper'
require 'carto/configuration'

# To avoid collisions with User model
module CartoDB
  # To avoid collisions with User class
  module UserModule
    class DBService
      include CartoDB::MiniSequel
      include Carto::Configuration
      extend CartoDB::SequelConnectionHelper

      # Also default schema for new users
      SCHEMA_PUBLIC = 'public'.freeze
      SCHEMA_CARTODB = 'cartodb'.freeze
      SCHEMA_IMPORTER = 'cdb_importer'.freeze
      SCHEMA_GEOCODING = 'cdb'.freeze
      SCHEMA_CDB_DATASERVICES_API = 'cdb_dataservices_client'.freeze
      SCHEMA_AGGREGATION_TABLES = 'aggregation'.freeze
      CDB_DATASERVICES_CLIENT_VERSION = '0.23.0'.freeze
      ODBC_FDW_VERSION = '0.2.0'.freeze

      def initialize(user)
        raise "User nil" unless user
        @user = user
        @queries = CartoDB::UserModule::DBQueries.new(@user)
      end

      def queries
        @queries
      end

      # This method is used both upon user creation and by the UserMover
      # All methods called inside should allow to be executed multiple times without errors
      def configure_database
        set_database_search_path

        grant_user_in_database

        set_statement_timeouts

        # INFO: Added to everyone because eases migration of normal users to org owners
        # and non-org users just don't use it
        set_user_as_organization_member

        if @user.database_schema == SCHEMA_PUBLIC
          setup_single_user_schema
        else
          setup_organization_user_schema
        end

        create_function_invalidate_varnish
        grant_publicuser_in_database
      end

      # All methods called inside should allow to be executed multiple times without errors
      def setup_single_user_schema
        set_user_privileges_at_db
        rebuild_quota_trigger
      end

      # All methods called inside should allow to be executed multiple times without errors
      def setup_organization_user_schema
        # WIP: CartoDB/cartodb-management#4467
        # Avoid mover reseting permissions. It's been moved to callers. Look for "WIP: CartoDB/cartodb-management#4467"
        # reset_user_schema_permissions
        reset_schema_owner
        set_user_privileges_at_db
        set_user_as_organization_member
        rebuild_quota_trigger

        # INFO: organization privileges are set for org_member_role, which is assigned to each org user
        if @user.organization_owner?
          setup_organization_owner
        end

        if @user.organization_admin?
          grant_admin_permissions
        end

        # Rebuild the geocoder api user config to reflect that is an organization user
        install_and_configure_geocoder_api_extension
        install_odbc_fdw
      end

      # INFO: main setup for non-org users
      def new_non_organization_user_main_db_setup
        Thread.new do
          create_db_user
          create_user_db
          grant_owner_in_database
        end.join
        create_importer_schema
        create_geocoding_schema
        load_cartodb_functions
        install_and_configure_geocoder_api_extension
        install_odbc_fdw
        # We reset the connections to this database to be sure the change in default search_path is effective
        reset_pooled_connections

        reset_database_permissions # Reset privileges

        # WIP: CartoDB/cartodb-management#4467
        # Added after commenting it in setup_organization_user_schema to avoid configure_database to reset permissions
        reset_user_schema_permissions

        configure_database

        revoke_cdb_conf_access
      end

      # INFO: main setup for org users
      def new_organization_user_main_db_setup
        Thread.new do
          create_db_user
        end.join
        create_own_schema

        # WIP: CartoDB/cartodb-management#4467
        # Added after commenting it in setup_organization_user_schema to avoid configure_database to reset permissions
        reset_user_schema_permissions

        setup_organization_user_schema
        install_and_configure_geocoder_api_extension
        # We reset the connections to this database to be sure the change in default search_path is effective
        reset_pooled_connections
        revoke_cdb_conf_access
      end

      def set_user_privileges_at_db # MU
        # INFO: organization permission on public schema is handled through role assignment
        unless @user.organization_user?
          set_user_privileges_in_cartodb_schema
          set_user_privileges_in_public_schema
        end

        set_user_privileges_in_own_schema
        set_privileges_to_publicuser_in_own_schema

        unless @user.organization_user?
          set_user_privileges_in_importer_schema
          set_user_privileges_in_geocoding_schema
          set_geo_columns_privileges
          set_raster_privileges
        end
      end

      def disable_writes
        # NOTE: This will not affect already opened connections. Run `terminate_database_conections` method after this
        # to ensure no more writes are possible.
        @user.in_database(as: :cluster_admin) do |database|
          database.run(%{
            ALTER DATABASE "#{@user.database_name}"
              SET default_transaction_read_only = 'on'
          })
        end
      end

      def enable_writes
        # NOTE: This will not affect already opened connections. Run `terminate_database_conections` method after this
        # to ensure no more writes are possible.
        @user.in_database(as: :cluster_admin) do |database|
          database.run(%{
            ALTER DATABASE "#{@user.database_name}"
              SET default_transaction_read_only = default
          })
        end
      end

      def writes_enabled?
        @user.in_database(as: :superuser) do |database|
          database.fetch(%{SHOW default_transaction_read_only}).first[:default_transaction_read_only] == 'off'
        end
      end

      # Cartodb functions
      def load_cartodb_functions(statement_timeout = nil, cdb_extension_target_version = nil)
        add_python

        # Install dependencies of cartodb extension
        @user.in_database(as: :superuser, no_cartodb_in_schema: true) do |db|
          db.transaction do
            unless statement_timeout.nil?
              old_timeout = db.fetch("SHOW statement_timeout;").first[:statement_timeout]
              db.run("SET statement_timeout TO '#{statement_timeout}';")
            end

            db.run('CREATE EXTENSION plpythonu FROM unpackaged') unless db.fetch(%{
                SELECT count(*) FROM pg_extension WHERE extname='plpythonu'
              }).first[:count] > 0
            db.run('CREATE EXTENSION postgis FROM unpackaged') unless db.fetch(%{
                SELECT count(*) FROM pg_extension WHERE extname='postgis'
              }).first[:count] > 0

            unless statement_timeout.nil?
              db.run("SET statement_timeout TO '#{old_timeout}';")
            end
          end
        end

        upgrade_cartodb_postgres_extension(statement_timeout, cdb_extension_target_version)

        rebuild_quota_trigger
      end

      def rebuild_quota_trigger
        @user.in_database(as: :superuser) do |db|
          rebuild_quota_trigger_with_database(db)
        end
      end

      def rebuild_quota_trigger_with_database(db)
        if !cartodb_extension_version_pre_mu? && @user.has_organization?
          db.run("DROP FUNCTION IF EXISTS public._CDB_UserQuotaInBytes();")
        end

        db.transaction do
          # NOTE: this has been written to work for both databases that switched to "cartodb" extension
          #       and those before the switch.
          #       In the future we should guarantee that exntension lives in cartodb schema so we don't need to set
          #       a search_path before
          search_path = db.fetch("SHOW search_path;").first[:search_path]
          db.run("SET search_path TO #{SCHEMA_CARTODB}, #{SCHEMA_PUBLIC};")
          if cartodb_extension_version_pre_mu?
            db.run("SELECT CDB_SetUserQuotaInBytes(#{@user.quota_in_bytes});")
          else
            db.run("SELECT CDB_SetUserQuotaInBytes('#{@user.database_schema}', #{@user.quota_in_bytes});")
          end

          # _CDB_UserQuotaInBytes is called by the quota trigger, and need to be open so
          # other users in the organization can run it (when updating shared datasets)
          db_user = @user.organization_user? ? organization_member_group_role_member_name : @user.database_username
          db.run(%{REVOKE ALL ON FUNCTION "#{@user.database_schema}"._CDB_UserQuotaInBytes() FROM PUBLIC;})
          db.run(%{GRANT ALL ON FUNCTION "#{@user.database_schema}"._CDB_UserQuotaInBytes() TO "#{db_user}";})

          db.run("SET search_path TO #{search_path};")
        end
      end

      def build_search_path(user_schema = nil, quote_user_schema = true)
        user_schema ||= @user.database_schema
        DBService.build_search_path(user_schema, quote_user_schema)
      end

      # Centralized method to provide the (ordered) search_path
      def self.build_search_path(user_schema, quote_user_schema = true)
        quote_char = quote_user_schema ? "\"" : ""
        "#{quote_char}#{user_schema}#{quote_char}, #{SCHEMA_CARTODB}, #{SCHEMA_CDB_DATASERVICES_API}, #{SCHEMA_PUBLIC}"
      end

      def set_database_search_path
        @user.in_database(as: :superuser) do |database|
          database.run(%{
            ALTER USER "#{@user.database_username}"
              SET search_path = #{build_search_path}
          })
        end
      end

      def create_importer_schema
        create_schema('cdb_importer')
      end

      def create_geocoding_schema
        create_schema('cdb')
      end

      def create_user_schema
        create_schema(@user.database_schema, @user.database_username)
      end

      def create_schema(schema, role = nil)
        @user.in_database(as: :superuser) do |db|
          Carto::Db::Database.new(@user.database_name, db).create_schema(schema, role)
        end
      end

      def grant_admin_permissions
        @user.in_database(as: :superuser) do |database|
          database.run(%{ SELECT cartodb.CDB_Organization_AddAdmin('#{@user.username}') })
        end
      end

      def revoke_admin_permissions
        @user.in_database(as: :superuser) do |database|
          database.run(%{ SELECT cartodb.CDB_Organization_RemoveAdmin('#{@user.username}') })
        end
      end

      def revoke_cdb_conf_access
        errors = []

        roles = [@user.database_username]
        if @user.organization_owner?
          begin
            roles << organization_member_group_role_member_name
          rescue => e
            errors << "WARN: Error fetching org member role (does #{@user.organization.name} has that role?)"
          end
        end
        roles << CartoDB::PUBLIC_DB_USER

        queries = []
        roles.map do |db_role|
          queries.concat(@queries.revoke_permissions_on_cartodb_conf_queries(db_role))
        end

        queries.map do |query|
          @user.in_database(as: :superuser) do |database|
            begin
              database.run(query)
            rescue => e
              # We can find organizations not yet upgraded for any reason or missing roles
              errors << e.message
            end
          end
        end

        errors
      rescue => e
        # For broken organizations
        ["FATAL ERROR for #{name}: #{e.message}"]
      end

      def create_public_db_user
        @user.in_database(as: :superuser) do |database|
          database.run(%{ CREATE USER "#{@user.database_public_username}" LOGIN INHERIT })
          database.run(%{ GRANT publicuser TO "#{@user.database_public_username}" })
          database.run(%{ ALTER USER "#{@user.database_public_username}" SET search_path = #{build_search_path} })
        end
      end

      def tables_effective(schema = 'public')
        @user.in_database do |user_database|
          user_database.synchronize do |conn|
            query = "select table_name::text from information_schema.tables where table_schema = '#{schema}'"
            tables = user_database[query].all.map { |i| i[:table_name] }
            return tables
          end
        end
      end

      def drop_owned_by_user(conn, role)
        conn.run("DROP OWNED BY \"#{role}\"")
      end

      def drop_user(conn = nil, username = nil)
        conn ||= @user.in_database(as: :cluster_admin)
        username ||= @user.database_username
        database_with_conflicts = nil
        retried = false

        begin
          conn.run("DROP USER IF EXISTS \"#{username}\"")
        rescue => e
          if !retried && e.message =~ /cannot be dropped because some objects depend on it/
            retried = true
            e.message =~ /object[s]? in database (.*)$/
            e.message =~ /privileges for database (.*)$/ unless $1
            if database_with_conflicts == $1
              raise e
            else
              database_with_conflicts = $1
              revoke_all_on_database_from(conn, database_with_conflicts, username)
              revoke_all_memberships_on_database_to_role(conn, username)
              drop_owned_by_user(conn, username)

              conflict_database_conn = @user.in_database(
                as: :cluster_admin,
                'database' => database_with_conflicts
              )
              drop_owned_by_user(conflict_database_conn, username)
              ['cdb', 'cdb_importer', 'cartodb', 'public', @user.database_schema].each do |s|
                drop_users_privileges_in_schema(s, [username], conn: conflict_database_conn)
              end
              DBService.close_sequel_connection(conflict_database_conn)
              retry
            end
          else
            raise e
          end
        end
      end

      # Org users share the same db, so must only delete the schema unless he's the owner
      def drop_organization_user(org_id, is_owner: false, force_destroy: false)
        raise CartoDB::BaseCartoDBError.new('Tried to delete an organization user without org id') if org_id.nil?
        Thread.new do
          @user.in_database(as: :superuser) do |database|
            if is_owner
              schemas = ['cdb', 'cdb_importer', 'cartodb', 'public', @user.database_schema] +
                        ::User.select(:database_schema).where(organization_id: org_id).all.collect(&:database_schema)
              schemas.uniq.each do |s|
                drop_users_privileges_in_schema(
                  s,
                  [@user.database_username, @user.database_public_username, CartoDB::PUBLIC_DB_USER])
              end

              # To avoid "cannot drop function" errors
              database.run("drop extension if exists plproxy cascade")
            end

            # If user is in an organization should never have public schema, but to be safe (& tests which stub stuff)
            unless @user.database_schema == SCHEMA_PUBLIC
              database.run(%{ DROP FUNCTION IF EXISTS "#{@user.database_schema}"._CDB_UserQuotaInBytes()})
              drop_analysis_cache(@user)
              drop_all_functions_from_schema(@user.database_schema)

              cascade_drop = force_destroy ? 'CASCADE' : ''
              database.run(%{ DROP SCHEMA IF EXISTS "#{@user.database_schema}" #{cascade_drop}})
            end
          end

          conn = @user.in_database(as: :cluster_admin)
          CartoDB::UserModule::DBService.terminate_database_connections(@user.database_name, @user.database_host)

          # If user is in an organization should never have public schema, but to be safe (& tests which stub stuff)
          unless @user.database_schema == SCHEMA_PUBLIC
            drop_user(conn, @user.database_public_username)
          end

          if is_owner
            conn.run("DROP DATABASE \"#{@user.database_name}\"")
          end
          drop_user(conn)
        end.join

        monitor_user_notification
      end

      def configure_extension_org_metadata_api_endpoint
        config = Cartodb.config[:org_metadata_api]
        host = config['host']
        port = config['port']
        username = config['username']
        password = config['password']
        timeout = config.fetch('timeout', 10)

        if host.present? && port.present? && username.present? && password.present?
          conf_sql = %{
            SELECT cartodb.CDB_Conf_SetConf('groups_api',
              '{ \"host\": \"#{host}\", \"port\": #{port}, \"timeout\": #{timeout}, \"username\": \"#{username}\",
                 \"password\": \"#{password}\"}'::json
            )
          }
          @user.in_database(as: :superuser) do |database|
            database.fetch(conf_sql).first
          end
        else
          CartoDB.notify_debug("org_metadata_api configuration missing", user_id: @user.id, config: config)
        end
      end

      def install_and_configure_geocoder_api_extension
          geocoder_api_config = Cartodb.get_config(:geocoder, 'api')
          # If there's no config we assume there's no need to install the
          # geocoder client as it is an independent API
          return if geocoder_api_config.blank?
          install_geocoder_api_extension(geocoder_api_config)
          @user.in_database(as: :superuser).run("ALTER USER \"#{@user.database_username}\"
              SET search_path TO #{build_search_path}")
          @user.in_database(as: :superuser).run("ALTER USER \"#{@user.database_public_username}\"
              SET search_path TO #{build_search_path}") if @user.organization_user?
          return true
        rescue => e
          CartoDB.notify_error(
            'Error installing and configuring geocoder api extension',
            error: e.inspect, user: @user
          )
          return false
      end

      def install_geocoder_api_extension(geocoder_api_config)
        @user.in_database(as: :superuser) do |db|
          db.transaction do
            db.run('CREATE EXTENSION IF NOT EXISTS plproxy SCHEMA public')
            db.run("CREATE EXTENSION IF NOT EXISTS cdb_dataservices_client VERSION '#{CDB_DATASERVICES_CLIENT_VERSION}'")
            db.run("ALTER EXTENSION cdb_dataservices_client UPDATE TO '#{CDB_DATASERVICES_CLIENT_VERSION}'")
            geocoder_server_sql = build_geocoder_server_config_sql(geocoder_api_config)
            db.run(geocoder_server_sql)
            db.run(build_entity_config_sql)
          end
        end
      end

      def install_odbc_fdw
        @user.in_database(as: :superuser) do |db|
          db.transaction do
            db.run('CREATE EXTENSION IF NOT EXISTS odbc_fdw SCHEMA public')
            db.run("ALTER EXTENSION odbc_fdw UPDATE TO '#{ODBC_FDW_VERSION}'")
          end
        end
      rescue Sequel::DatabaseError
        # For the time being we'll be resilient to the odbc_fdw not being available
        # and just proceed without installing it.
      end

      def setup_organization_owner
        setup_organization_role_permissions
        configure_extension_org_metadata_api_endpoint
      end

      # Use a direct connection to the db through the direct port specified
      # in the database configuration and set up its statement timeout value. This
      # allows to overpass the statement_timeout limit if a connection pooler is used.
      # This method is supposed to receive a block that will be run with the created
      # connection.
      def in_database_direct_connection(statement_timeout:)
        raise 'need block' unless block_given?

        configuration = db_configuration_for
        configuration[:port] = configuration.fetch(:direct_port, configuration["direct_port"]) || configuration[:port] || configuration["port"]

        # Temporary trace to be removed once https://github.com/CartoDB/cartodb/issues/7047 is solved
        CartoDB::Logger.warning(message: 'Direct connection not used from queue') unless Socket.gethostname =~ /^que/

        connection = @user.get_connection(_opts = {}, configuration)

        begin
          connection.run("SET statement_timeout TO #{statement_timeout}")
          yield(connection)
        ensure
          connection.run("SET statement_timeout TO DEFAULT")
        end
      end

      def reset_pooled_connections
        # Only close connections to this users' database
        $pool.close_connections!(@user.database_name)
      end

      # Upgrade the cartodb postgresql extension
      def upgrade_cartodb_postgres_extension(statement_timeout = nil, cdb_extension_target_version = nil)
        if cdb_extension_target_version.nil?
          cdb_extension_target_version = '0.20.0'
        end

        @user.in_database(as: :superuser, no_cartodb_in_schema: true) do |db|
          db.transaction do
            unless statement_timeout.nil?
              old_timeout = db.fetch("SHOW statement_timeout;").first[:statement_timeout]
              db.run("SET statement_timeout TO '#{statement_timeout}';")
            end

            db.run(%{
              DO LANGUAGE 'plpgsql' $$
              DECLARE
                ver TEXT;
              BEGIN
                BEGIN
                  SELECT cartodb.cdb_version() INTO ver;
                EXCEPTION WHEN undefined_function OR invalid_schema_name THEN
                  RAISE NOTICE 'Got % (%)', SQLERRM, SQLSTATE;
                  BEGIN
                    CREATE EXTENSION cartodb VERSION '#{cdb_extension_target_version}' FROM unpackaged;
                  EXCEPTION WHEN undefined_table THEN
                    RAISE NOTICE 'Got % (%)', SQLERRM, SQLSTATE;
                    CREATE EXTENSION cartodb VERSION '#{cdb_extension_target_version}';
                    RETURN;
                  END;
                  RETURN;
                END;
                ver := '#{cdb_extension_target_version}';
                IF position('dev' in ver) > 0 THEN
                  EXECUTE 'ALTER EXTENSION cartodb UPDATE TO ''' || ver || 'next''';
                  EXECUTE 'ALTER EXTENSION cartodb UPDATE TO ''' || ver || '''';
                ELSE
                  EXECUTE 'ALTER EXTENSION cartodb UPDATE TO ''' || ver || '''';
                END IF;
              END;
              $$;
            })

            unless statement_timeout.nil?
              db.run("SET statement_timeout TO '#{old_timeout}';")
            end

            obtained = db.fetch('SELECT cartodb.cdb_version() as v').first[:v]

            unless cartodb_extension_semver(cdb_extension_target_version) == cartodb_extension_semver(obtained)
              raise("Expected cartodb extension '#{cdb_extension_target_version}' obtained '#{obtained}'")
            end
          end
        end
      end

      def cartodb_extension_version_pre_mu?
        current_version = cartodb_extension_semver(cartodb_extension_version)
        if current_version.size == 3
          major, minor, = current_version
          major == 0 && minor < 3
        else
          raise 'Current cartodb extension version does not match standard x.y.z format'
        end
      end

      # Returns a tree elements array with [major, minor, patch] as in http://semver.org/
      def cartodb_extension_semver(extension_version)
        extension_version.split('.').take(3).map(&:to_i)
      end

      def cartodb_extension_version
        @cartodb_extension_version ||= @user.in_database(as: :superuser)
                                            .fetch('SELECT cartodb.cdb_version() AS v').first[:v]
      end

      def reset_user_schema_permissions
        @user.in_database(as: :superuser) do |user_database|
          user_database.transaction do
            schemas = [@user.database_schema].uniq
            schemas.each do |schema|
              revoke_privileges(user_database, schema, 'PUBLIC')
            end
            yield(user_database) if block_given?
          end
        end
      end

      def reset_database_permissions
        @user.in_database(as: :superuser) do |user_database|
          user_database.transaction do
            schemas = %w(public cdb_importer cdb cartodb)
            ['PUBLIC', CartoDB::PUBLIC_DB_USER].each do |user|
              revoke_all_on_database_from(user_database, @user.database_name, user)
              schemas.each do |schema|
                revoke_privileges(user_database, schema, user)
              end
            end
            yield(user_database) if block_given?
          end
        end
      end

      def set_statement_timeouts
        @user.in_database(as: :superuser) do |user_database|
          user_database["ALTER ROLE \"?\" SET statement_timeout to ?", @user.database_username.lit,
                        @user.user_timeout].all
          user_database["ALTER DATABASE \"?\" SET statement_timeout to ?", @user.database_name.lit,
                        @user.database_timeout].all
        end
        @user.in_database.disconnect
        @user.in_database.connect(db_configuration_for)
        @user.in_database(as: :public_user).disconnect
        @user.in_database(as: :public_user).connect(db_configuration_for(:public_user))
      rescue Sequel::DatabaseConnectionError
      end

      def set_user_as_organization_member
        @user.in_database(as: :superuser) do |user_database|
          user_database.transaction do
            user_database.run("SELECT cartodb.CDB_Organization_Create_Member('#{@user.database_username}');")
          end
        end
      end

      def reset_schema_owner
        @user.in_database(as: :superuser) do |database|
          database.run(%{ ALTER SCHEMA "#{@user.database_schema}" OWNER TO "#{@user.database_username}" })
        end
      end

      def grant_user_in_database
        @queries.run_in_transaction(
          @queries.grant_connect_on_database_queries,
          true
        )
      end

      def grant_publicuser_in_database
        @queries.run_in_transaction(
          @queries.grant_connect_on_database_queries(CartoDB::PUBLIC_DB_USER),
          true
        )
        @queries.run_in_transaction(
          @queries.grant_read_on_schema_queries(SCHEMA_CARTODB, CartoDB::PUBLIC_DB_USER),
          true
        )
        @queries.run_in_transaction(
          [
            "REVOKE SELECT ON cartodb.cdb_tablemetadata FROM #{CartoDB::PUBLIC_DB_USER} CASCADE",
            "REVOKE SELECT ON cartodb.cdb_analysis_catalog FROM #{CartoDB::PUBLIC_DB_USER} CASCADE"
          ],
          true
        )
        @queries.run_in_transaction(
          [
            "GRANT USAGE ON SCHEMA #{SCHEMA_PUBLIC} TO #{CartoDB::PUBLIC_DB_USER}",
            "GRANT EXECUTE ON ALL FUNCTIONS IN SCHEMA #{SCHEMA_PUBLIC} TO #{CartoDB::PUBLIC_DB_USER}",
            "GRANT SELECT ON spatial_ref_sys TO #{CartoDB::PUBLIC_DB_USER}"
          ],
          true
        )
      end

      def set_user_privileges_in_own_schema # MU
        @queries.run_in_transaction(
          @queries.grant_all_on_user_schema_queries,
          true
        )
      end

      def set_user_privileges_in_cartodb_schema(db_user = nil)
        @queries.run_in_transaction(
          (
            @queries.grant_read_on_schema_queries(SCHEMA_CARTODB, db_user) +
            @queries.grant_write_on_cdb_tablemetadata_queries(db_user) +
            @queries.grant_write_on_cdb_analysis_catalog_queries(db_user)
          ),
          true
        )
      end

      def set_privileges_to_publicuser_in_own_schema # MU
        # Privileges in user schema for publicuser
        @queries.run_in_transaction(
          @queries.grant_usage_on_user_schema_to_other(CartoDB::PUBLIC_DB_USER),
          true
        )
      end

      def set_user_privileges_in_public_schema(db_user = nil)
        @queries.run_in_transaction(
          @queries.grant_read_on_schema_queries(SCHEMA_PUBLIC, db_user),
          true
        )
      end

      def set_user_privileges_in_importer_schema(db_user = nil) # MU
        @queries.run_in_transaction(
          @queries.grant_all_on_schema_queries(SCHEMA_IMPORTER, db_user),
          true
        )
      end

      def revoke_all_on_database_from(conn, database, role)
        conn.run("REVOKE ALL ON DATABASE \"#{database}\" FROM \"#{role}\" CASCADE") if role_exists?(conn, role)
      end

      def grant_owner_in_database
        @queries.run_in_transaction(
          @queries.grant_all_on_database_queries,
          true
        )
      end

      def fix_table_permissions
        tables_queries = []
        @user.tables.each do |table|
          if table.public? || table.public_with_link_only?
            tables_queries << %{
              GRANT SELECT ON \"#{@user.database_schema}\".\"#{table.name}\" TO #{CartoDB::PUBLIC_DB_USER} }
          end
          tables_queries << %{
            ALTER TABLE \"#{@user.database_schema}\".\"#{table.name}\" OWNER TO \"#{@user.database_username}\" }
        end
        @queries.run_in_transaction(
          tables_queries,
          true
        )
      end

      def set_user_privileges_in_geocoding_schema(db_user = nil)
        @queries.run_in_transaction(
          @queries.grant_all_on_schema_queries(SCHEMA_GEOCODING, db_user),
          true
        )
      end

      def set_geo_columns_privileges(role_name = nil)
        # Postgis lives at public schema, as do geometry_columns and geography_columns
        catalogs_schema = SCHEMA_PUBLIC
        target_user = role_name.nil? ? @user.database_public_username : role_name
        queries = [
          %{ GRANT SELECT ON "#{catalogs_schema}"."geometry_columns" TO "#{target_user}" },
          %{ GRANT SELECT ON "#{catalogs_schema}"."geography_columns" TO "#{target_user}" }
        ]
        @queries.run_in_transaction(queries, true)
      end

      def set_raster_privileges(role_name = nil)
        # Postgis lives at public schema, so raster catalogs too
        catalogs_schema = SCHEMA_PUBLIC
        queries = [
          "GRANT SELECT ON TABLE \"#{catalogs_schema}\".\"raster_overviews\" TO \"#{CartoDB::PUBLIC_DB_USER}\"",
          "GRANT SELECT ON TABLE \"#{catalogs_schema}\".\"raster_columns\" TO \"#{CartoDB::PUBLIC_DB_USER}\""
        ]
        target_user = role_name.nil? ? @user.database_public_username : role_name
        unless @user.organization.nil?
          queries << "GRANT SELECT ON TABLE \"#{catalogs_schema}\".\"raster_overviews\" TO \"#{target_user}\""
          queries << "GRANT SELECT ON TABLE \"#{catalogs_schema}\".\"raster_columns\" TO \"#{target_user}\""
        end
        @queries.run_in_transaction(queries, true)
      end

      def setup_organization_role_permissions
        org_member_role = organization_member_group_role_member_name
        set_user_privileges_in_public_schema(org_member_role)
        @queries.run_in_transaction(
          @queries.grant_connect_on_database_queries(org_member_role), true
        )
        set_geo_columns_privileges(org_member_role)
        set_raster_privileges(org_member_role)
        set_user_privileges_in_cartodb_schema(org_member_role)
        set_user_privileges_in_importer_schema(org_member_role)
        set_user_privileges_in_geocoding_schema(org_member_role)
      end

      def schema_exists?(schema, database = @user.in_database)
        query = "SELECT 1 as schema_exist FROM information_schema.schemata WHERE schema_name = '#{schema}'"
        !database.fetch(query).first.nil?
      end

      def drop_users_privileges_in_schema(schema, accounts, conn: nil)
        user_database = conn || @user.in_database(as: :superuser, statement_timeout: 600000)
        return unless schema_exists?(schema, user_database)

        user_database.transaction do
          accounts
            .select { |role| role_exists?(user_database, role) }
            .each { |role| revoke_privileges(user_database, schema, "\"#{role}\"") }
        end
      end

      def revoke_all_memberships_on_database_to_role(conn, role)
        conn.fetch(%{
            select rolname from pg_user join pg_auth_members on (pg_user.usesysid=pg_auth_members.member)
            join pg_roles on (pg_roles.oid=pg_auth_members.roleid) where pg_user.usename='#{role}'
          }).each do |rolname|
          conn.run("REVOKE \"#{rolname[:rolname]}\" FROM \"#{role}\" CASCADE")
        end
      end

      def drop_analysis_cache(user)
        # Filtering this query by tableowner should be enough but for security reasons I've added an additional
        # filter by schema. Also we have add a regexp to matches the current analysis tables name format to avoid
        # deleting user tables in the process (https://github.com/CartoDB/camshaft/blob/0.59.4/lib/node/node.js#L344-L348)
        cache_tables_sql = "SELECT tablename FROM pg_tables WHERE schemaname = '#{user.database_schema}' and tableowner = '#{user.database_username}' and tablename ~* '^analysis_[0-9a-z]{10}_[0-9a-z]{40}$';"
        delete_analysis_metadata_sql = "DELETE FROM cdb_analysis_catalog WHERE username = '#{user.username}'"
        user.in_database(as: :superuser) do |database|
          database.transaction do
            database.fetch(cache_tables_sql).map(:tablename).each do |cache_table_name|
              database.run(%{DROP TABLE "#{user.database_schema}"."#{cache_table_name}"})
            end
            database.run(delete_analysis_metadata_sql)
          end
        end
      end

      # Drops grants and functions in a given schema, avoiding by all means a CASCADE
      # to not affect extensions or other users
      def drop_all_functions_from_schema(schema_name)
        recursivity_max_depth = 3

        return if schema_name == SCHEMA_PUBLIC

        @user.in_database(as: :superuser) do |database|
          # Non-aggregate functions
          drop_function_sqls = database.fetch(%{
            SELECT 'DROP FUNCTION "' || ns.nspname || '".' || proname || '(' || oidvectortypes(proargtypes) || ');'
              AS sql
            FROM pg_proc INNER JOIN pg_namespace ns ON (pg_proc.pronamespace = ns.oid AND pg_proc.proisagg = FALSE)
            WHERE ns.nspname = '#{schema_name}'
          })

          # Simulate a controlled environment drop cascade contained to only functions
          failed_sqls = []
          recursivity_level = 0
          begin
            failed_sqls = []
            drop_function_sqls.each do |sql_sentence|
              begin
                database.run(sql_sentence[:sql])
              rescue Sequel::DatabaseError => e
                if e.message =~ /depends on function /i
                  failed_sqls.push(sql_sentence)
                else
                  raise
                end
              end
            end
            drop_function_sqls = failed_sqls
            recursivity_level += 1
          end while failed_sqls.count > 0 && recursivity_level < recursivity_max_depth

          # If something remains, reattempt later after dropping aggregates
          if drop_function_sqls.count > 0
            aggregate_dependant_function_sqls = drop_function_sqls
          else
            aggregate_dependant_function_sqls = []
          end

          # And now aggregate functions
          failed_sqls = []
          drop_function_sqls = database.fetch(%{
            SELECT 'DROP AGGREGATE ' || ns.nspname || '.' || proname || '(' || oidvectortypes(proargtypes) || ');'
              AS sql
            FROM pg_proc INNER JOIN pg_namespace ns ON (pg_proc.pronamespace = ns.oid AND pg_proc.proisagg = TRUE)
            WHERE ns.nspname = '#{schema_name}'
          })
          drop_function_sqls.each do |sql_sentence|
            begin
              database.run(sql_sentence[:sql])
            rescue Sequel::DatabaseError
              failed_sqls.push(sql_sentence)
            end
          end

          if failed_sqls.count > 0
            raise CartoDB::BaseCartoDBError.new('Cannot drop schema aggregate functions, dependencies remain')
          end

          # One final pass of normal functions, if left
          if aggregate_dependant_function_sqls.count > 0
            aggregate_dependant_function_sqls.each do |sql_sentence|
              begin
                database.run(sql_sentence[:sql])
              rescue Sequel::DatabaseError
                failed_sqls.push(sql_sentence)
              end
            end
          end

          if failed_sqls.count > 0
            raise CartoDB::BaseCartoDBError.new('Cannot drop schema functions, dependencies remain')
          end
        end
      end

      # Create a "public.cdb_invalidate_varnish()" function to invalidate Varnish
      #
      # The function can only be used by the superuser, we expect
      # security-definer triggers OR triggers on superuser-owned tables
      # to call it with controlled set of parameters.
      #
      # The function is written in python because it needs to reach out
      # to a Varnish server.
      #
      # Being unable to communicate with Varnish may or may not be critical
      # depending on CartoDB configuration at time of function definition.
      #
      def create_function_invalidate_varnish
        if Cartodb.config[:invalidation_service] && Cartodb.config[:invalidation_service].fetch('enabled', false)
          create_function_invalidate_varnish_invalidation_service
        elsif Cartodb.config[:varnish_management].fetch('http_port', false)
          create_function_invalidate_varnish_http
        else
          create_function_invalidate_varnish_telnet
        end
      end

      # Add plpythonu pl handler
      def add_python
        @user.in_database(
          as: :superuser,
          no_cartodb_in_schema: true
        ).run("CREATE OR REPLACE PROCEDURAL LANGUAGE 'plpythonu' HANDLER plpython_call_handler;")
      end

      # Needed because in some cases it might not exist and failure ends transaction
      def role_exists?(db, role)
        !db.fetch("SELECT 1 FROM pg_roles WHERE rolname='#{role}'").first.nil?
      end

      def revoke_privileges(db, schema, user)
        db.run("REVOKE ALL ON SCHEMA \"#{schema}\" FROM #{user} CASCADE")
        db.run("REVOKE ALL ON ALL SEQUENCES IN SCHEMA \"#{schema}\" FROM #{user} CASCADE")
        db.run("REVOKE ALL ON ALL FUNCTIONS IN SCHEMA \"#{schema}\" FROM #{user} CASCADE")
        db.run("REVOKE ALL ON ALL TABLES IN SCHEMA \"#{schema}\" FROM #{user} CASCADE")
      end

      def organization_member_group_role_member_name
        @user.in_database.fetch(
          "SELECT cartodb.CDB_Organization_Member_Group_Role_Member_Name() as org_member_role;"
        )[:org_member_role][:org_member_role]
      end

      def db_configuration_for(user_role = nil)
        logger = (Rails.env.development? || Rails.env.test? ? ::Rails.logger : nil)
        if user_role == :superuser
          ::SequelRails.configuration.environment_for(Rails.env).merge(
            'database' => @user.database_name,
            :logger => logger,
            'host' => @user.database_host
          ) { |_, o, n| n.nil? ? o : n }
        elsif user_role == :cluster_admin
          ::SequelRails.configuration.environment_for(Rails.env).merge(
            'database' => 'postgres',
            :logger => logger,
            'host' => @user.database_host
          ) { |_, o, n| n.nil? ? o : n }
        elsif user_role == :public_user
          ::SequelRails.configuration.environment_for(Rails.env).merge(
            'database' => @user.database_name,
            :logger => logger,
            'username' => CartoDB::PUBLIC_DB_USER, 'password' => CartoDB::PUBLIC_DB_USER_PASSWORD,
            'host' => @user.database_host
          ) { |_, o, n| n.nil? ? o : n }
        elsif user_role == :public_db_user
          ::SequelRails.configuration.environment_for(Rails.env).merge(
            'database' => @user.database_name,
            :logger => logger,
            'username' => @user.database_public_username, 'password' => CartoDB::PUBLIC_DB_USER_PASSWORD,
            'host' => @user.database_host
          ) { |_, o, n| n.nil? ? o : n }
        else
          ::SequelRails.configuration.environment_for(Rails.env).merge(
            'database' => @user.database_name,
            :logger => logger,
            'username' => @user.database_username,
            'password' => @user.database_password,
            'host' => @user.database_host
          ) { |_, o, n| n.nil? ? o : n }
        end
      end

      def monitor_user_notification
        FileUtils.touch(log_file_path('users_modifications'))
        if !Cartodb.config[:signups].nil? && !Cartodb.config[:signups]["service"].nil? &&
           !Cartodb.config[:signups]["service"]["port"].nil?
          enable_remote_db_user
        end
      end

      def enable_remote_db_user
        request = http_client.request(
          "#{@user.database_host}:#{Cartodb.config[:signups]['service']['port']}/scripts/activate_db_user",
          method: :post,
          headers: { "Content-Type" => "application/json" }
        )
        response = request.run
        if response.code != 200
          raise(response.body)
        else
          comm_response = JSON.parse(response.body)
          if comm_response['retcode'].to_i != 0
            raise(response['stderr'])
          end
        end
      end

      def create_own_schema
        load_cartodb_functions
        @user.database_schema = @user.username
        @user.this.update(database_schema: @user.database_schema)
        create_user_schema
        set_database_search_path
        create_public_db_user
      end

      def move_to_own_schema
        new_schema_name = @user.username
        old_database_schema_name = @user.database_schema
        if @user.database_schema != new_schema_name
          Carto::Db::UserSchemaMover.new(@user).move_objects(new_schema_name)

          create_public_db_user
          set_database_search_path
        end
      rescue => e
        # Undo metadata changes if process fails
        begin
          @user.this.update database_schema: old_database_schema_name

          # Defensive measure to avoid undesired table dropping
          if schema_exists?(new_schema_name) && tables_effective(new_schema_name).count == 0
            drop_all_functions_from_schema(new_schema_name)
            @user.in_database.run(%{ DROP SCHEMA "#{new_schema_name}" })
          end
        rescue => ee
          # Avoid shadowing the actual error
          CartoDB.notify_exception(ee, user: @user)
        end
        raise e
      end

      def drop_database_and_user(conn = nil)
        conn ||= @user.in_database(as: :cluster_admin)

        if !@user.database_name.nil? && !@user.database_name.empty?
          @user.in_database(as: :superuser).run("DROP SCHEMA \"#{@user.database_schema}\" CASCADE")
          conn.run("UPDATE pg_database SET datallowconn = 'false' WHERE datname = '#{@user.database_name}'")
          CartoDB::UserModule::DBService.terminate_database_connections(@user.database_name, @user.database_host)
          conn.run("DROP DATABASE \"#{@user.database_name}\"")
        end

        if !@user.database_username.nil? && !@user.database_username.empty?
          conn.run("DROP USER \"#{@user.database_username}\"")
        end
      end

      def run_pg_query(query)
        time = nil
        res  = nil
        translation_proc = nil
        @user.in_database do |user_database|
          time = Benchmark.measure do
            user_database.synchronize do |conn|
              res = conn.exec query
            end
            translation_proc = user_database.conversion_procs
          end
        end
        {
          time: time.real,
          total_rows: res.ntuples,
          rows: pg_to_hash(res, translation_proc),
          results: pg_results?(res),
          modified: pg_modified?(res),
          affected_rows: pg_size(res)
        }
      rescue => e
        if e.is_a? PGError
          if e.message.include?("does not exist")
            if e.message.include?("column")
              raise CartoDB::ColumnNotExists, e.message
            else
              raise CartoDB::TableNotExists, e.message
            end
          else
            raise CartoDB::ErrorRunningQuery, e.message
          end
        else
          raise e
        end
      end

      def create_db_user
        conn = @user.in_database(as: :cluster_admin)
        begin
          conn.transaction do
            begin
              conn.run("CREATE USER \"#{@user.database_username}\" PASSWORD '#{@user.database_password}'")
              conn.run("GRANT publicuser to \"#{@user.database_username}\"")
            rescue => e
              puts "#{Time.now} USER SETUP ERROR (#{@user.database_username}): #{$!}"
              raise e
            end
          end
        end
      end

      def create_user_db
        conn = @user.in_database(as: :cluster_admin)
        begin
          conn.run("CREATE DATABASE \"#{@user.database_name}\"
          WITH TEMPLATE = template_postgis
          OWNER = #{::SequelRails.configuration.environment_for(Rails.env)['username']}
          ENCODING = 'UTF8'
          CONNECTION LIMIT=-1")
        rescue => e
          puts "#{Time.now} USER SETUP ERROR WHEN CREATING DATABASE #{@user.database_name}: #{$!}"
          raise e
        end
      end

      def set_database_name
        @user.database_name = case Rails.env
                              when 'development'
                                "cartodb_dev_user_#{@user.partial_db_name}_db"
                              when 'staging'
                                "cartodb_staging_user_#{@user.partial_db_name}_db"
                              when 'test'
                                "cartodb_test_user_#{@user.partial_db_name}_db"
                              else
                                "cartodb_user_#{@user.partial_db_name}_db"
                              end
        if @user.has_organization_enabled?
          if !@user.database_exists?
            raise "Organization database #{@user.database_name} doesn't exist"
          end
        else
          if @user.database_exists?
            raise "Database #{@user.database_name} already exists"
          end
        end
        @user.this.update database_name: @user.database_name
      end

      def public_user_roles
        @user.organization_user? ? [CartoDB::PUBLIC_DB_USER, @user.database_public_username] : [CartoDB::PUBLIC_DB_USER]
      end

      def terminate_database_connections
        CartoDB::UserModule::DBService.terminate_database_connections(@user.database_name, @user.database_host)
      end

      def self.terminate_database_connections(database_name, database_host)
        connection_params = ::SequelRails.configuration.environment_for(Rails.env).merge(
          'host' => database_host,
          'database' => 'postgres'
        ) { |_, o, n| n.nil? ? o : n }
        conn = ::Sequel.connect(connection_params)
        conn.run("
          DO language plpgsql $$
          DECLARE
              ver INT;
              sql TEXT;
          BEGIN
              SELECT INTO ver setting from pg_settings where name='server_version_num';
              sql := 'SELECT pg_terminate_backend(';
              IF ver > 90199 THEN
                sql := sql || 'pid';
              ELSE
                sql := sql || 'procpid';
              END IF;

              sql := sql || ') FROM pg_stat_activity WHERE datname = '
                || quote_literal('#{database_name}');

              RAISE NOTICE '%', sql;

              EXECUTE sql;
          END
          $$
        ")
        close_sequel_connection(conn)
      end

      def triggers(schema = @user.database_schema)
        Carto::Db::Database.build_with_user(@user).triggers(schema)
      end

      def functions(schema = @user.database_schema)
        Carto::Db::Database.build_with_user(@user).functions(schema, @user.database_username)
      end

      def views(schema = @user.database_schema)
        Carto::Db::Database.build_with_user(@user).views(schema, @user.database_username)
      end

      def materialized_views(schema = @user.database_schema)
        Carto::Db::Database.build_with_user(@user).materialized_views(schema, @user.database_username)
      end

      def get_database_version
        version_match = @user.in_database.fetch("SELECT version()").first[:version].match(/(PostgreSQL (([0-9]+\.?){2,3})).*/)
        if version_match.nil?
          return nil
        else
          return version_match[2]
        end
      end

      def connect_to_aggregation_tables
        config = Cartodb.get_config(:aggregation_tables)
        @user.in_database(as: :superuser) do |db|
          db.transaction do
            db.run(build_aggregation_fdw_config_sql(config))
            db.run("SELECT cartodb._CDB_Setup_FDW('aggregation');")
            db.run("CREATE FOREIGN TABLE IF NOT EXISTS #{SCHEMA_AGGREGATION_TABLES}.agg_admin0 " \
                   "(cartodb_id integer, the_geom geometry(Geometry,4326), " \
                   "the_geom_webmercator geometry(Geometry,3857), " \
                   "population double precision OPTIONS (column_name 'pop_est')) SERVER aggregation OPTIONS " \
                   "(schema_name 'public', table_name '#{config['tables']['admin0']}', updatable 'false');")
            db.run("CREATE FOREIGN TABLE IF NOT EXISTS #{SCHEMA_AGGREGATION_TABLES}.agg_admin1 " \
                   "(cartodb_id integer,the_geom geometry(Geometry,4326), " \
                   "the_geom_webmercator geometry(Geometry,3857)) " \
                   "SERVER aggregation OPTIONS (schema_name 'public', table_name '#{config['tables']['admin1']}', updatable 'false');")
            db.run("GRANT SELECT ON TABLE #{SCHEMA_AGGREGATION_TABLES}.agg_admin0 TO \"#{@user.database_username}\";")
            db.run("GRANT SELECT ON TABLE #{SCHEMA_AGGREGATION_TABLES}.agg_admin1 TO \"#{@user.database_username}\";")
          end
        end
      end

      # Execute a query in the user database
      # @param [String] query, using $1, $2 ... for placeholders
      # @param ... values for the placeholders
      # @return [Array<Hash<String, Value>>] Something (an actual Array in this case) that behaves like an Array
      #                                      of Hashes that map column name (as string) to value
      # @raise [PG::Error] if the query fails
      def execute_in_user_database(query, *binds)
        placeholder_query = query.gsub(/\$\d+/, '?')
        @user.in_database.fetch(placeholder_query, *binds).all.map(&:stringify_keys)
      rescue Sequel::DatabaseError => exception
        raise exception.cause
      end

      private

      def http_client
        @http_client ||= Carto::Http::Client.get('old_user', log_requests: true)
      end

      # Telnet invalidation works only for Varnish 2.x.
      def create_function_invalidate_varnish_telnet
        add_python

        varnish_host = Cartodb.config[:varnish_management].try(:[], 'host') || '127.0.0.1'
        varnish_port = Cartodb.config[:varnish_management].try(:[], 'port') || 6082
        varnish_timeout = Cartodb.config[:varnish_management].try(:[], 'timeout') || 5
        varnish_critical = Cartodb.config[:varnish_management].try(:[], 'critical') == true ? 1 : 0
        varnish_retry = Cartodb.config[:varnish_management].try(:[], 'retry') || 5
        purge_command = Cartodb::config[:varnish_management]["purge_command"]
        varnish_trigger_verbose = Cartodb.config[:varnish_management].fetch('trigger_verbose', true) == true ? 1 : 0

        @user.in_database(as: :superuser).run(
          <<-TRIGGER
            BEGIN;
            CREATE OR REPLACE FUNCTION public.cdb_invalidate_varnish(table_name text) RETURNS void AS
            $$
                critical = #{varnish_critical}
                timeout = #{varnish_timeout}
                retry = #{varnish_retry}
                trigger_verbose = #{varnish_trigger_verbose}

                client = GD.get('varnish', None)
                for i in ('base64', 'hashlib'):
                  if not i in GD:
                    GD[i] = __import__(i)

                while True:

                  if not client:
                      try:
                        import varnish
                        client = GD['varnish'] = varnish.VarnishHandler(('#{varnish_host}', #{varnish_port}, timeout))
                      except Exception as err:
                        # NOTE: we won't retry on connection error
                        if critical:
                          plpy.error('Varnish connection error: ' +  str(err))
                        break

                  try:
                    cache_key = "t:" + GD['base64'].b64encode(GD['hashlib'].sha256('#{@user.database_name}:%s' % table_name).digest())[0:6]
                    # We want to say \b here, but the Varnish telnet interface expects \\b, we have to escape that on Python to \\\\b and double that for SQL
                    client.fetch('#{purge_command} obj.http.Surrogate-Key ~ "\\\\\\\\b%s\\\\\\\\b"' % cache_key)
                    break
                  except Exception as err:
                    if trigger_verbose:
                      plpy.warning('Varnish fetch error: ' + str(err))
                    client = GD['varnish'] = None # force reconnect
                    if not retry:
                      if critical:
                        plpy.error('Varnish fetch error: ' +  str(err))
                      break
                    retry -= 1 # try reconnecting
            $$
            LANGUAGE 'plpythonu' VOLATILE;
            REVOKE ALL ON FUNCTION public.cdb_invalidate_varnish(TEXT) FROM PUBLIC;
            COMMIT;
        TRIGGER
        )
      end

      def create_function_invalidate_varnish_http
        add_python

        varnish_host = Cartodb.config[:varnish_management].try(:[], 'host') || '127.0.0.1'
        varnish_port = Cartodb.config[:varnish_management].try(:[], 'http_port') || 6081
        varnish_timeout = Cartodb.config[:varnish_management].try(:[], 'timeout') || 5
        varnish_critical = Cartodb.config[:varnish_management].try(:[], 'critical') == true ? 1 : 0
        varnish_retry = Cartodb.config[:varnish_management].try(:[], 'retry') || 5
        varnish_trigger_verbose = Cartodb.config[:varnish_management].fetch('trigger_verbose', true) == true ? 1 : 0

        @user.in_database(as: :superuser).run(
          <<-TRIGGER
            BEGIN;
            CREATE OR REPLACE FUNCTION public.cdb_invalidate_varnish(table_name text) RETURNS void AS
            $$
                critical = #{varnish_critical}
                timeout = #{varnish_timeout}
                retry = #{varnish_retry}
                trigger_verbose = #{varnish_trigger_verbose}
                for i in ('httplib', 'base64', 'hashlib'):
                  if not i in GD:
                    GD[i] = __import__(i)

                while True:

                  try:
                    client = GD['httplib'].HTTPConnection('#{varnish_host}', #{varnish_port}, False, timeout)
                    cache_key = "t:" + GD['base64'].b64encode(GD['hashlib'].sha256('#{@user.database_name}:%s' % table_name).digest())[0:6]
                    client.request('PURGE', '/key', '', {"Invalidation-Match": ('\\\\b%s\\\\b' % cache_key) })
                    response = client.getresponse()
                    assert response.status == 204
                    break
                  except Exception as err:
                    if trigger_verbose:
                      plpy.warning('Varnish purge error: ' + str(err))
                    if not retry:
                      if critical:
                        plpy.error('Varnish purge error: ' +  str(err))
                      break
                    retry -= 1 # try reconnecting
            $$
            LANGUAGE 'plpythonu' VOLATILE;
            REVOKE ALL ON FUNCTION public.cdb_invalidate_varnish(TEXT) FROM PUBLIC;
            COMMIT;
          TRIGGER
        )
      end

      # Invalidate through external service
      def create_function_invalidate_varnish_invalidation_service
        add_python

        invalidation_host = Cartodb.config[:invalidation_service].try(:[], 'host') || '127.0.0.1'
        invalidation_port = Cartodb.config[:invalidation_service].try(:[], 'port') || 3142
        invalidation_timeout = Cartodb.config[:invalidation_service].try(:[], 'timeout') || 5
        invalidation_critical = Cartodb.config[:invalidation_service].try(:[], 'critical') ? 1 : 0
        invalidation_retry = Cartodb.config[:invalidation_service].try(:[], 'retry') || 5
        invalidation_trigger_verbose =
          Cartodb.config[:invalidation_service].fetch('trigger_verbose', true) == true ? 1 : 0

        @user.in_database(as: :superuser).run(
          <<-TRIGGER
            BEGIN;
            CREATE OR REPLACE FUNCTION public.cdb_invalidate_varnish(table_name text) RETURNS void AS
            $$
                critical = #{invalidation_critical}
                timeout = #{invalidation_timeout}
                retry = #{invalidation_retry}
                trigger_verbose = #{invalidation_trigger_verbose}

                client = GD.get('invalidation', None)

                if 'syslog' not in GD:
                  import syslog
                  GD['syslog'] = syslog
                else:
                  syslog = GD['syslog']

                if 'time' not in GD:
                  import time
                  GD['time'] = time
                else:
                  time = GD['time']

                if 'json' not in GD:
                  import json
                  GD['json'] = json
                else:
                  json = GD['json']

                start = time.time()
                retries = 0
                termination_state = 1
                error = ''

                while True:

                  if not client:
                      try:
                        import redis
                        client = GD['invalidation'] = redis.Redis(host='#{invalidation_host}', port=#{invalidation_port}, socket_timeout=timeout)
                      except Exception as err:
                        error = "client_error - %s" % str(err)
                        # NOTE: we won't retry on connection error
                        if critical:
                          plpy.error('Invalidation Service connection error: ' +  str(err))
                        break

                  try:
                    client.execute_command('TCH', '#{@user.database_name}', table_name)
                    termination_state = 0
                    error = ''
                    break
                  except Exception as err:
                    error = "request_error - %s" % str(err)
                    if trigger_verbose:
                      plpy.warning('Invalidation Service warning: ' + str(err))
                    client = GD['invalidation'] = None # force reconnect
                    if not retry:
                      if critical:
                        plpy.error('Invalidation Service error: ' +  str(err))
                      break
                    retries = retries + 1
                    retry -= 1 # try reconnecting

                end = time.time()
                invalidation_duration = (end - start)
                current_time = time.strftime("%Y-%m-%d %H:%M:%S %Z", time.localtime())
                session_user = plpy.execute("SELECT session_user", 1)[0]["session_user"]
                invalidation_result = {"timestamp": current_time, "duration": round(invalidation_duration, 8), "termination_state": termination_state, "retries": retries, "error": error, "database": "#{@user.database_name}", "table_name": table_name, "dbuser": session_user}

                if trigger_verbose:
                  syslog.syslog(syslog.LOG_INFO, "invalidation: %s" % json.dumps(invalidation_result))

            $$
            LANGUAGE 'plpythonu' VOLATILE;
            REVOKE ALL ON FUNCTION public.cdb_invalidate_varnish(TEXT) FROM PUBLIC;
            COMMIT;
          TRIGGER
        )
      end

      # Geocoder api extension related
      def build_geocoder_server_config_sql(config)
        host = config['host']
        port = config['port']
        user = config['user']
        dbname = config['dbname']
        %{
          SELECT cartodb.CDB_Conf_SetConf('geocoder_server_config',
            '{ \"connection_str\": \"host=#{host} port=#{port} dbname=#{dbname} user=#{user}\"}'::json
          );
        }
      end

      def build_entity_config_sql
        # User configuration
        entity_name = @user.organization_user? ? @user.organization.name : @user.username
        %{
          SELECT cartodb.CDB_Conf_SetConf('user_config',
            '{"is_organization": #{@user.organization_user?}, "entity_name": "#{entity_name}"}'::json
          );
        }
      end

      def build_aggregation_fdw_config_sql(config)
        %{
          SELECT cartodb.CDB_Conf_SetConf('fdws',
            '{"aggregation":{"server":{"extensions":"postgis", "dbname":"#{config['dbname']}",
            "host":"#{config['host']}", "port":"#{config['port']}"}, "users":{"public":{"user":"#{config['username']}",
            "password":"#{config['password']}"} } } }'::json
          );
        }
      end
    end
  end
end


text:
package org.apache.flink.api.io.avro;

import org.apache.flink.api.io.avro.generated.Address;
import org.apache.flink.api.io.avro.generated.Colors;
import org.apache.flink.api.io.avro.generated.Fixed16;
import org.apache.flink.api.io.avro.generated.User;
import org.apache.flink.api.java.io.AvroInputFormat;
import org.apache.flink.api.java.tuple.Tuple2;
import org.apache.flink.configuration.Configuration;
import org.apache.flink.core.fs.FileInputSplit;
import org.apache.flink.core.fs.Path;

import org.apache.avro.file.DataFileWriter;
import org.apache.avro.io.DatumWriter;
import org.apache.avro.specific.SpecificDatumWriter;
import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;

import java.io.File;
import java.io.IOException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.Random;

import static org.junit.Assert.assertEquals;

/**
 * Test the avro input format.
 * (The testcase is mostly the getting started tutorial of avro)
 * http://avro.apache.org/docs/current/gettingstartedjava.html
 */
public class AvroSplittableInputFormatTest {

	private File testFile;

	static final String TEST_NAME = "Alyssa";

	static final String TEST_ARRAY_STRING_1 = "ELEMENT 1";
	static final String TEST_ARRAY_STRING_2 = "ELEMENT 2";

	static final boolean TEST_ARRAY_BOOLEAN_1 = true;
	static final boolean TEST_ARRAY_BOOLEAN_2 = false;

	static final Colors TEST_ENUM_COLOR = Colors.GREEN;

	static final String TEST_MAP_KEY1 = "KEY 1";
	static final long TEST_MAP_VALUE1 = 8546456L;
	static final String TEST_MAP_KEY2 = "KEY 2";
	static final long TEST_MAP_VALUE2 = 17554L;

	static final Integer TEST_NUM = new Integer(239);
	static final String TEST_STREET = "Baker Street";
	static final String TEST_CITY = "London";
	static final String TEST_STATE = "London";
	static final String TEST_ZIP = "NW1 6XE";

	static final int NUM_RECORDS = 5000;

	@Before
	public void createFiles() throws IOException {
		testFile = File.createTempFile("AvroSplittableInputFormatTest", null);

		ArrayList<CharSequence> stringArray = new ArrayList<CharSequence>();
		stringArray.add(TEST_ARRAY_STRING_1);
		stringArray.add(TEST_ARRAY_STRING_2);

		ArrayList<Boolean> booleanArray = new ArrayList<Boolean>();
		booleanArray.add(TEST_ARRAY_BOOLEAN_1);
		booleanArray.add(TEST_ARRAY_BOOLEAN_2);

		HashMap<CharSequence, Long> longMap = new HashMap<CharSequence, Long>();
		longMap.put(TEST_MAP_KEY1, TEST_MAP_VALUE1);
		longMap.put(TEST_MAP_KEY2, TEST_MAP_VALUE2);

		Address addr = new Address();
		addr.setNum(new Integer(TEST_NUM));
		addr.setStreet(TEST_STREET);
		addr.setCity(TEST_CITY);
		addr.setState(TEST_STATE);
		addr.setZip(TEST_ZIP);

		User user1 = new User();
		user1.setName(TEST_NAME);
		user1.setFavoriteNumber(256);
		user1.setTypeDoubleTest(123.45d);
		user1.setTypeBoolTest(true);
		user1.setTypeArrayString(stringArray);
		user1.setTypeArrayBoolean(booleanArray);
		user1.setTypeEnum(TEST_ENUM_COLOR);
		user1.setTypeMap(longMap);
		user1.setTypeNested(addr);

		// Construct via builder
		User user2 = User.newBuilder()
				.setName(TEST_NAME)
				.setFavoriteColor("blue")
				.setFavoriteNumber(null)
				.setTypeBoolTest(false)
				.setTypeDoubleTest(1.337d)
				.setTypeNullTest(null)
				.setTypeLongTest(1337L)
				.setTypeArrayString(new ArrayList<CharSequence>())
				.setTypeArrayBoolean(new ArrayList<Boolean>())
				.setTypeNullableArray(null)
				.setTypeEnum(Colors.RED)
				.setTypeMap(new HashMap<CharSequence, Long>())
				.setTypeFixed(new Fixed16())
				.setTypeUnion(123L)
				.setTypeNested(
						Address.newBuilder().setNum(TEST_NUM).setStreet(TEST_STREET)
								.setCity(TEST_CITY).setState(TEST_STATE).setZip(TEST_ZIP)
								.build())
				.build();
		DatumWriter<User> userDatumWriter = new SpecificDatumWriter<User>(User.class);
		DataFileWriter<User> dataFileWriter = new DataFileWriter<User>(userDatumWriter);
		dataFileWriter.create(user1.getSchema(), testFile);
		dataFileWriter.append(user1);
		dataFileWriter.append(user2);

		Random rnd = new Random(1337);
		for (int i = 0; i < NUM_RECORDS - 2; i++) {
			User user = new User();
			user.setName(TEST_NAME + rnd.nextInt());
			user.setFavoriteNumber(rnd.nextInt());
			user.setTypeDoubleTest(rnd.nextDouble());
			user.setTypeBoolTest(true);
			user.setTypeArrayString(stringArray);
			user.setTypeArrayBoolean(booleanArray);
			user.setTypeEnum(TEST_ENUM_COLOR);
			user.setTypeMap(longMap);
			Address address = new Address();
			address.setNum(new Integer(TEST_NUM));
			address.setStreet(TEST_STREET);
			address.setCity(TEST_CITY);
			address.setState(TEST_STATE);
			address.setZip(TEST_ZIP);
			user.setTypeNested(address);

			dataFileWriter.append(user);
		}
		dataFileWriter.close();
	}

	@Test
	public void testSplittedIF() throws IOException {
		Configuration parameters = new Configuration();

		AvroInputFormat<User> format = new AvroInputFormat<User>(new Path(testFile.getAbsolutePath()), User.class);

		format.configure(parameters);
		FileInputSplit[] splits = format.createInputSplits(4);
		assertEquals(splits.length, 4);
		int elements = 0;
		int[] elementsPerSplit = new int[4];
		for (int i = 0; i < splits.length; i++) {
			format.open(splits[i]);
			while (!format.reachedEnd()) {
				User u = format.nextRecord(null);
				Assert.assertTrue(u.getName().toString().startsWith(TEST_NAME));
				elements++;
				elementsPerSplit[i]++;
			}
			format.close();
		}

		Assert.assertEquals(1539, elementsPerSplit[0]);
		Assert.assertEquals(1026, elementsPerSplit[1]);
		Assert.assertEquals(1539, elementsPerSplit[2]);
		Assert.assertEquals(896, elementsPerSplit[3]);
		Assert.assertEquals(NUM_RECORDS, elements);
		format.close();
	}

	@Test
	public void testAvroRecoveryWithFailureAtStart() throws Exception {
		final int recordsUntilCheckpoint = 132;

		Configuration parameters = new Configuration();

		AvroInputFormat<User> format = new AvroInputFormat<User>(new Path(testFile.getAbsolutePath()), User.class);
		format.configure(parameters);

		FileInputSplit[] splits = format.createInputSplits(4);
		assertEquals(splits.length, 4);

		int elements = 0;
		int[] elementsPerSplit = new int[4];
		for (int i = 0; i < splits.length; i++) {
			format.reopen(splits[i], format.getCurrentState());
			while (!format.reachedEnd()) {
				User u = format.nextRecord(null);
				Assert.assertTrue(u.getName().toString().startsWith(TEST_NAME));
				elements++;

				if (format.getRecordsReadFromBlock() == recordsUntilCheckpoint) {

					// do the whole checkpoint-restore procedure and see if we pick up from where we left off.
					Tuple2<Long, Long> state = format.getCurrentState();

					// this is to make sure that nothing stays from the previous format
					// (as it is going to be in the normal case)
					format = new AvroInputFormat<>(new Path(testFile.getAbsolutePath()), User.class);

					format.reopen(splits[i], state);
					assertEquals(format.getRecordsReadFromBlock(), recordsUntilCheckpoint);
				}
				elementsPerSplit[i]++;
			}
			format.close();
		}

		Assert.assertEquals(1539, elementsPerSplit[0]);
		Assert.assertEquals(1026, elementsPerSplit[1]);
		Assert.assertEquals(1539, elementsPerSplit[2]);
		Assert.assertEquals(896, elementsPerSplit[3]);
		Assert.assertEquals(NUM_RECORDS, elements);
		format.close();
	}

	@Test
	public void testAvroRecovery() throws Exception {
		final int recordsUntilCheckpoint = 132;

		Configuration parameters = new Configuration();

		AvroInputFormat<User> format = new AvroInputFormat<User>(new Path(testFile.getAbsolutePath()), User.class);
		format.configure(parameters);

		FileInputSplit[] splits = format.createInputSplits(4);
		assertEquals(splits.length, 4);

		int elements = 0;
		int[] elementsPerSplit = new int[4];
		for (int i = 0; i < splits.length; i++) {
			format.open(splits[i]);
			while (!format.reachedEnd()) {
				User u = format.nextRecord(null);
				Assert.assertTrue(u.getName().toString().startsWith(TEST_NAME));
				elements++;

				if (format.getRecordsReadFromBlock() == recordsUntilCheckpoint) {

					// do the whole checkpoint-restore procedure and see if we pick up from where we left off.
					Tuple2<Long, Long> state = format.getCurrentState();

					// this is to make sure that nothing stays from the previous format
					// (as it is going to be in the normal case)
					format = new AvroInputFormat<>(new Path(testFile.getAbsolutePath()), User.class);

					format.reopen(splits[i], state);
					assertEquals(format.getRecordsReadFromBlock(), recordsUntilCheckpoint);
				}
				elementsPerSplit[i]++;
			}
			format.close();
		}

		Assert.assertEquals(1539, elementsPerSplit[0]);
		Assert.assertEquals(1026, elementsPerSplit[1]);
		Assert.assertEquals(1539, elementsPerSplit[2]);
		Assert.assertEquals(896, elementsPerSplit[3]);
		Assert.assertEquals(NUM_RECORDS, elements);
		format.close();
	}

	/*
	This test is gave the reference values for the test of Flink's IF.

	This dependency needs to be added

        <dependency>
            <groupId>org.apache.avro</groupId>
            <artifactId>avro-mapred</artifactId>
            <version>1.7.6</version>
        </dependency>

	@Test
	public void testHadoop() throws Exception {
		JobConf jf = new JobConf();
		FileInputFormat.addInputPath(jf, new org.apache.hadoop.fs.Path(testFile.toURI()));
		jf.setBoolean(org.apache.avro.mapred.AvroInputFormat.IGNORE_FILES_WITHOUT_EXTENSION_KEY, false);
		org.apache.avro.mapred.AvroInputFormat<User> format = new org.apache.avro.mapred.AvroInputFormat<User>();
		InputSplit[] sp = format.getSplits(jf, 4);
		int elementsPerSplit[] = new int[4];
		int cnt = 0;
		int i = 0;
		for (InputSplit s:sp) {
			RecordReader<AvroWrapper<User>, NullWritable> r = format.getRecordReader(s, jf, new HadoopDummyReporter());
			AvroWrapper<User> k = r.createKey();
			NullWritable v = r.createValue();

			while (r.next(k, v)) {
				cnt++;
				elementsPerSplit[i]++;
			}
			i++;
		}
		System.out.println("Status "+Arrays.toString(elementsPerSplit));
	} **/

	@After
	public void deleteFiles() {
		testFile.delete();
	}
}

redpajama_set_name: RedPajamaGithub


text:
module Moonstone
  class Tokenizer < Lucene::Analysis::Tokenizer

    include Lucene::Analysis

    def initialize(reader)
      @reader = java.io.BufferedReader.new(reader)
    end

    # No, this is not terribly useful.  Subclass me already.
    def next(token=nil)
      token = (token ? token.clear :  Token.new)
      token.set_term_text @reader.read_line
      token.set_start_offset 1
      token.set_end_offset 1
    end

  end
end


text:
package org.apache.ambari.server.orm.dao;

import java.text.MessageFormat;
import java.util.Collection;
import java.util.Collections;
import java.util.List;

import javax.persistence.EntityManager;
import javax.persistence.TypedQuery;

import org.apache.ambari.server.actionmanager.HostRoleStatus;
import org.apache.ambari.server.orm.RequiresSession;
import org.apache.ambari.server.orm.entities.RequestEntity;
import org.apache.ambari.server.orm.entities.RequestResourceFilterEntity;
import org.eclipse.persistence.config.HintValues;
import org.eclipse.persistence.config.QueryHints;

import com.google.inject.Inject;
import com.google.inject.Provider;
import com.google.inject.Singleton;
import com.google.inject.persist.Transactional;

@Singleton
public class RequestDAO {
  /**
   * SQL template to retrieve all request IDs, sorted by the ID.
   */
  private final static String REQUEST_IDS_SORTED_SQL = "SELECT request.requestId FROM RequestEntity request ORDER BY request.requestId {0}";

  @Inject
  Provider<EntityManager> entityManagerProvider;

  @Inject
  DaoUtils daoUtils;

  @RequiresSession
  public RequestEntity findByPK(Long requestId) {
    return entityManagerProvider.get().find(RequestEntity.class, requestId);
  }

  @RequiresSession
  public List<RequestEntity> findByPks(Collection<Long> requestIds) {
    return findByPks(requestIds, false);
  }

  /**
   * Given a collection of request ids, load the corresponding entities
   * @param requestIds  the collection of request ids
   * @param refreshHint {@code true} to hint JPA that the list should be refreshed
   * @return the list entities. An empty list if the requestIds are not provided
   */
  @RequiresSession
  public List<RequestEntity> findByPks(Collection<Long> requestIds, boolean refreshHint) {
    if (null == requestIds || 0 == requestIds.size()) {
      return Collections.emptyList();
    }

    TypedQuery<RequestEntity> query = entityManagerProvider.get().createQuery("SELECT request FROM RequestEntity request " +
        "WHERE request.requestId IN ?1", RequestEntity.class);

    // !!! https://bugs.eclipse.org/bugs/show_bug.cgi?id=398067
    // ensure that an associated entity with a JOIN is not stale
    if (refreshHint) {
      query.setHint(QueryHints.REFRESH, HintValues.TRUE);
    }

    return daoUtils.selectList(query, requestIds);
  }

  @RequiresSession
  public List<RequestEntity> findAll() {
    return daoUtils.selectAll(entityManagerProvider.get(), RequestEntity.class);
  }

  @RequiresSession
  public List<Long> findAllRequestIds(int limit, boolean ascending) {
    String sort = "ASC";
    if (!ascending) {
      sort = "DESC";
    }

    String sql = MessageFormat.format(REQUEST_IDS_SORTED_SQL, sort);
    TypedQuery<Long> query = entityManagerProvider.get().createQuery(sql,
        Long.class);

    query.setMaxResults(limit);

    return daoUtils.selectList(query);
  }

  @RequiresSession
  public List<RequestResourceFilterEntity> findAllResourceFilters() {
    return daoUtils.selectAll(entityManagerProvider.get(), RequestResourceFilterEntity.class);
  }

  @RequiresSession
  public boolean isAllTasksCompleted(long requestId) {
    TypedQuery<Long> query = entityManagerProvider.get().createQuery(
        "SELECT task.taskId FROM HostRoleCommandEntity task WHERE task.requestId = ?1 AND " +
          "task.stageId=(select max(stage.stageId) FROM StageEntity stage WHERE stage.requestId=?1) " +
          "AND task.status NOT IN ?2",
        Long.class
    );
    query.setMaxResults(1); //we don't need all
    return daoUtils.selectList(query, requestId, HostRoleStatus.getCompletedStates()).isEmpty();
  }

  @RequiresSession
  public Long getLastStageId(long requestId) {
    TypedQuery<Long> query = entityManagerProvider.get().createQuery("SELECT max(stage.stageId) " +
      "FROM StageEntity stage WHERE stage.requestId=?1", Long.class);
    return daoUtils.selectSingle(query, requestId);
  }

  @Transactional
  public void create(RequestEntity requestEntity) {
    entityManagerProvider.get().persist(requestEntity);
  }

  @Transactional
  public RequestEntity merge(RequestEntity requestEntity) {
    return entityManagerProvider.get().merge(requestEntity);
  }

  @Transactional
  public void remove(RequestEntity requestEntity) {
    entityManagerProvider.get().remove(merge(requestEntity));
  }

  @Transactional
  public void removeByPK(Long requestId) {
    remove(findByPK(requestId));
  }
}


text:
import React, { PropTypes } from 'react';
import { Link, IndexLink } from 'react-router';

// This is a class-based component because the current
// version of hot reloading won't hot reload a stateless
// component at the top-level.
class App extends React.Component {
  render() {
    return (

        <IndexLink to="/">Home</IndexLink>
        {' | '}
        <Link to="/fuel-savings">Demo App</Link>
        {' | '}
        <Link to="/about">About</Link>

        {this.props.children}

    );
  }
}

App.propTypes = {
  children: PropTypes.element
};

export default App;


text:
<?php

/* FDARestautixBundle:Restautix:byebye.html.twig */
class __TwigTemplate_128c9449d653c79779bde17155482b1b extends Twig_Template
{
    public function __construct(Twig_Environment $env)
    {
        parent::__construct($env);

        $this->parent = false;

        $this->blocks = array(
        );
    }

    protected function doDisplay(array $context, array $blocks = array())
    {
        // line 1
        echo "<!DOCTYPE html>

        <title>Bievenue sur la page de Restautix, l'application pour gérer vos repas entre collègues :-)</title>

Bye bye ";
        // line 7
        echo twig_escape_filter($this->env, (isset($context["nom"]) ? $context["nom"] : $this->getContext($context, "nom")), "html", null, true);
        echo " !

            Ceci n'est pas la fin ^^

";
    }

    public function getTemplateName()
    {
        return "FDARestautixBundle:Restautix:byebye.html.twig";
    }

    public function isTraitable()
    {
        return false;
    }

    public function getDebugInfo()
    {
        return array (  27 => 7,  19 => 1,);
    }
}


text:
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
    <modelVersion>4.0.0</modelVersion>

    <groupId>ru.bragnikita.serverview</groupId>
    <artifactId>web</artifactId>
    <version>0.0.1-SNAPSHOT</version>
    <packaging>war</packaging>
    <properties>
        <org.springframework-version>4.1.0.RELEASE</org.springframework-version>
    </properties>
    <dependencies>
        <dependency>
            <groupId>org.springframework</groupId>
            <artifactId>spring-webmvc</artifactId>
            <version>${org.springframework-version}</version>
            <exclusions>
                <exclusion>
                    <groupId>commons-logging</groupId>
                    <artifactId>commons-logging</artifactId>
                </exclusion>
            </exclusions>
        </dependency>
        <dependency>
            <groupId>com.google.guava</groupId>
            <artifactId>guava</artifactId>
            <version>18.0</version>
        </dependency>
        <dependency>
            <groupId>org.apache.commons</groupId>
            <artifactId>commons-lang3</artifactId>
            <version>3.2.1</version>
        </dependency>
        <dependency>
            <groupId>org.slf4j</groupId>
            <artifactId>jcl-over-slf4j</artifactId>
            <version>1.7.5</version>
            <scope>runtime</scope>
        </dependency>
        <dependency>
            <groupId>ch.qos.logback</groupId>
            <artifactId>logback-classic</artifactId>
            <version>1.0.7</version>
        </dependency>
        <dependency>
            <groupId>com.fasterxml.jackson.core</groupId>
            <artifactId>jackson-core</artifactId>
            <version>2.3.0</version>
        </dependency>
        <!--Utility dependencies -->
        <dependency>
            <groupId>commons-io</groupId>
            <artifactId>commons-io</artifactId>
            <version>2.4</version>
        </dependency>
        <dependency>
            <groupId>joda-time</groupId>
            <artifactId>joda-time</artifactId>
            <version>1.6.2</version>
        </dependency>
        <!-- Test dependencies -->
        <dependency>
            <groupId>org.testng</groupId>
            <artifactId>testng</artifactId>
            <version>6.8.7</version>
            <scope>test</scope>
        </dependency>
        <dependency>
            <groupId>org.springframework</groupId>
            <artifactId>spring-test</artifactId>
            <version>${org.springframework-version}</version>
            <scope>test</scope>
        </dependency>
        <dependency>
            <groupId>com.jayway.jsonpath</groupId>
            <artifactId>json-path</artifactId>
            <version>0.9.1</version>
            <scope>test</scope>
        </dependency>
        <dependency>
            <groupId>groovy</groupId>
            <artifactId>groovy-all</artifactId>
            <version>1.7.5</version>
            <scope>test</scope>
        </dependency>
        <!--<dependency>-->
            <!--<groupId>org.spockframework</groupId>-->
            <!--<artifactId>spock-core</artifactId>-->
            <!--<version>1.0-groovy-2.3-SNAPSHOT</version>-->
            <!--<scope>test</scope>-->
        <!--</dependency>-->
        <dependency>
            <groupId>org.spockframework</groupId>
            <artifactId>spock-spring</artifactId>
            <version>0.7-groovy-2.0</version>
            <scope>test</scope>
        </dependency>
        <dependency>
            <groupId>com.fasterxml.jackson.core</groupId>
            <artifactId>jackson-databind</artifactId>
            <version>2.3.0</version>
        </dependency>
        <dependency>
            <groupId>javax.servlet</groupId>
            <artifactId>javax.servlet-api</artifactId>
            <version>3.1.0</version>
            <scope>provided</scope>
        </dependency>

    </dependencies>
    <repositories>
        <repository>
            <id>spock-snapshots</id>
            <url>http://oss.sonatype.org/content/repositories/snapshots/</url>
            <snapshots>
                <enabled>true</enabled>
            </snapshots>
        </repository>
    </repositories>
    <build>
        <finalName>serverview-${project.version}</finalName>
    </build>
</project>


text:
    <states>
        <state name="aboutme"></state>
        <state name="mywork"></state>
        <state name="resume"></state>
    </states>


text:
require 'redis'
require 'redis-namespace'
require 'eventmachine'
require 'json'
require 'logger'

# Periodically scrapes Icinga's logs, and stores this information in Redis.
class GaloisScraper
  attr_accessor :config, :logger, :redis, :icinga_status, :prd

  # @param [Hash] config
  # @option config [Array] :host_fields The fields to be scraped from each host
  # @option config [Array] :service_fields The fields to be scraped from each service
  # @option config [String] :icinga_config An absolute path to Icinga's "icinga.cfg" file.
  # @option config [Integer] :refresh The amount of time, in seconds, between log scrapings.
  # @param [Logger] logger An instance of Logger which GaloisEmailer will write to.
  def initialize(config, logger = Logger.new(nil))
    @config = {
        :host_fields  => [],
        :service_fields  => []
      }.merge config

    @logger = logger

    @redis = {
      :fd => Redis.current,
      :ns => Redis::Namespace.new(:galois, :redis => Redis.current)
    }

    File.open(@config[:icinga_config]).each{ |line| @icinga_status = $1 if line.match(/status_file=(.*)/) }.close
    File.open(@config[:prd]) {|file| @prd = JSON.parse(file.read)}

  rescue
    @logger.error("Had some trouble with initialization:\n#{$!}")
  end

  # Begin scraping
  def start
    parse
    EM.run { EM.add_periodic_timer(@config[:refresh]) {parse} }
  end

  # Scrape the log file
  def parse
    status = File.open(@icinga_status) {|file| file.read}

    @redis[:ns].del("entities")
    parse_entity(status, "hoststatus", @config[:host_fields], &method(:add_prd))
    parse_entity(status, "servicestatus", @config[:service_fields], &method(:add_prd))

  rescue
    @logger.error("There were some issues with parsing and persisting the status data:\n#{$!}\n#{$@}")
  end

  # Parse all entities is the source and add them to Redis
  # @param [String] source The string to be parsed
  # @param [String] entity_name The name of the entity to be parsed
  # @param [Array] fields A list of the fields to be extracted from each entity
  def parse_entity(source, entity_name, fields)
    source.scan(/#{entity_name} {(.*?)}/m).flatten.each do |entity|
      index = @redis[:ns].hincrby("entities", "#{entity_name}", 1) - 1
      key = "#{entity_name}::#{index}"

      @redis[:ns].del(key)
      fields.each { |field| if value = parse_field(entity, field) then @redis[:ns].hmset(key, field, value) end }

      yield key if block_given?
    end
  end

  # Extract the value from the field
  # @param [String] entity The string to be parsed
  # @param [String] field The name of the field whose value will be extracted
  def parse_field(entity, field)
    field_name = entity.match(/#{field}=(.*)/) {$1}

  ensure
    @logger.warn("Found no field named #{field} in entity:\n#{entity}") if field_name.nil?
  end

  # Add a field to the entity given by key which indicates if the entity is on a production machine.
  # @param [String] key The redis key of the entity
  def add_prd(key)
    @redis[:ns].hset(key, "isPrd?", @prd.include?(@redis[:ns].hget(key, "host_name")))
  end

end


text:
#ifndef _LIBS_UTILS_MUTEX_H
#define _LIBS_UTILS_MUTEX_H

#include <stdint.h>
#include <sys/types.h>
#include <time.h>

#if !defined(_WIN32)
# include <pthread.h>
#endif

#include <utils/Errors.h>
#include <utils/Timers.h>

// Enable thread safety attributes only with clang.
// The attributes can be safely erased when compiling with other compilers.
#if defined(__clang__) && (!defined(SWIG))
#define THREAD_ANNOTATION_ATTRIBUTE__(x) __attribute__((x))
#else
#define THREAD_ANNOTATION_ATTRIBUTE__(x)  // no-op
#endif

#define CAPABILITY(x) THREAD_ANNOTATION_ATTRIBUTE__(capability(x))

#define SCOPED_CAPABILITY THREAD_ANNOTATION_ATTRIBUTE__(scoped_lockable)

#define GUARDED_BY(x) THREAD_ANNOTATION_ATTRIBUTE__(guarded_by(x))

#define PT_GUARDED_BY(x) THREAD_ANNOTATION_ATTRIBUTE__(pt_guarded_by(x))

#define ACQUIRED_BEFORE(...) THREAD_ANNOTATION_ATTRIBUTE__(acquired_before(__VA_ARGS__))

#define ACQUIRED_AFTER(...) THREAD_ANNOTATION_ATTRIBUTE__(acquired_after(__VA_ARGS__))

#define REQUIRES(...) THREAD_ANNOTATION_ATTRIBUTE__(requires_capability(__VA_ARGS__))

#define REQUIRES_SHARED(...) THREAD_ANNOTATION_ATTRIBUTE__(requires_shared_capability(__VA_ARGS__))

#define ACQUIRE(...) THREAD_ANNOTATION_ATTRIBUTE__(acquire_capability(__VA_ARGS__))

#define ACQUIRE_SHARED(...) THREAD_ANNOTATION_ATTRIBUTE__(acquire_shared_capability(__VA_ARGS__))

#define RELEASE(...) THREAD_ANNOTATION_ATTRIBUTE__(release_capability(__VA_ARGS__))

#define RELEASE_SHARED(...) THREAD_ANNOTATION_ATTRIBUTE__(release_shared_capability(__VA_ARGS__))

#define TRY_ACQUIRE(...) THREAD_ANNOTATION_ATTRIBUTE__(try_acquire_capability(__VA_ARGS__))

#define TRY_ACQUIRE_SHARED(...) \
    THREAD_ANNOTATION_ATTRIBUTE__(try_acquire_shared_capability(__VA_ARGS__))

#define EXCLUDES(...) THREAD_ANNOTATION_ATTRIBUTE__(locks_excluded(__VA_ARGS__))

#define ASSERT_CAPABILITY(x) THREAD_ANNOTATION_ATTRIBUTE__(assert_capability(x))

#define ASSERT_SHARED_CAPABILITY(x) THREAD_ANNOTATION_ATTRIBUTE__(assert_shared_capability(x))

#define RETURN_CAPABILITY(x) THREAD_ANNOTATION_ATTRIBUTE__(lock_returned(x))

#define NO_THREAD_SAFETY_ANALYSIS THREAD_ANNOTATION_ATTRIBUTE__(no_thread_safety_analysis)

// ---------------------------------------------------------------------------
namespace android {
// ---------------------------------------------------------------------------

class Condition;

/*
 * NOTE: This class is for code that builds on Win32.  Its usage is
 * deprecated for code which doesn't build for Win32.  New code which
 * doesn't build for Win32 should use std::mutex and std::lock_guard instead.
 *
 * Simple mutex class.  The implementation is system-dependent.
 *
 * The mutex must be unlocked by the thread that locked it.  They are not
 * recursive, i.e. the same thread can't lock it multiple times.
 */
class CAPABILITY("mutex") Mutex {
  public:
    enum {
        PRIVATE = 0,
        SHARED = 1
    };

    Mutex();
    explicit Mutex(const char* name);
    explicit Mutex(int type, const char* name = nullptr);
    ~Mutex();

    // lock or unlock the mutex
    status_t lock() ACQUIRE();
    void unlock() RELEASE();

    // lock if possible; returns 0 on success, error otherwise
    status_t tryLock() TRY_ACQUIRE(0);

#if defined(__ANDROID__)
    // Lock the mutex, but don't wait longer than timeoutNs (relative time).
    // Returns 0 on success, TIMED_OUT for failure due to timeout expiration.
    //
    // OSX doesn't have pthread_mutex_timedlock() or equivalent. To keep
    // capabilities consistent across host OSes, this method is only available
    // when building Android binaries.
    //
    // FIXME?: pthread_mutex_timedlock is based on CLOCK_REALTIME,
    // which is subject to NTP adjustments, and includes time during suspend,
    // so a timeout may occur even though no processes could run.
    // Not holding a partial wakelock may lead to a system suspend.
    status_t timedLock(nsecs_t timeoutNs) TRY_ACQUIRE(0);
#endif

    // Manages the mutex automatically. It'll be locked when Autolock is
    // constructed and released when Autolock goes out of scope.
    class SCOPED_CAPABILITY Autolock {
      public:
        inline explicit Autolock(Mutex& mutex) ACQUIRE(mutex) : mLock(mutex) { mLock.lock(); }
        inline explicit Autolock(Mutex* mutex) ACQUIRE(mutex) : mLock(*mutex) { mLock.lock(); }
        inline ~Autolock() RELEASE() { mLock.unlock(); }

      private:
        Mutex& mLock;
        // Cannot be copied or moved - declarations only
        Autolock(const Autolock&);
        Autolock& operator=(const Autolock&);
    };

  private:
    friend class Condition;

    // A mutex cannot be copied
    Mutex(const Mutex&);
    Mutex& operator=(const Mutex&);

#if !defined(_WIN32)
    pthread_mutex_t mMutex;
#else
    void _init();
    void* mState;
#endif
};

// ---------------------------------------------------------------------------

#if !defined(_WIN32)

inline Mutex::Mutex() {
    pthread_mutex_init(&mMutex, nullptr);
}
inline Mutex::Mutex(__attribute__((unused)) const char* name) {
    pthread_mutex_init(&mMutex, nullptr);
}
inline Mutex::Mutex(int type, __attribute__((unused)) const char* name) {
    if (type == SHARED) {
        pthread_mutexattr_t attr;
        pthread_mutexattr_init(&attr);
        pthread_mutexattr_setpshared(&attr, PTHREAD_PROCESS_SHARED);
        pthread_mutex_init(&mMutex, &attr);
        pthread_mutexattr_destroy(&attr);
    } else {
        pthread_mutex_init(&mMutex, nullptr);
    }
}
inline Mutex::~Mutex() {
    pthread_mutex_destroy(&mMutex);
}
inline status_t Mutex::lock() {
    return -pthread_mutex_lock(&mMutex);
}
inline void Mutex::unlock() {
    pthread_mutex_unlock(&mMutex);
}
inline status_t Mutex::tryLock() {
    return -pthread_mutex_trylock(&mMutex);
}
#if defined(__ANDROID__)
inline status_t Mutex::timedLock(nsecs_t timeoutNs) {
    timeoutNs += systemTime(SYSTEM_TIME_REALTIME);
    const struct timespec ts = {
        /* .tv_sec = */ static_cast<time_t>(timeoutNs / 1000000000),
        /* .tv_nsec = */ static_cast<long>(timeoutNs % 1000000000),
    };
    return -pthread_mutex_timedlock(&mMutex, &ts);
}
#endif

#endif // !defined(_WIN32)

// ---------------------------------------------------------------------------

/*
 * Automatic mutex.  Declare one of these at the top of a function.
 * When the function returns, it will go out of scope, and release the
 * mutex.
 */

typedef Mutex::Autolock AutoMutex;

// ---------------------------------------------------------------------------
}  // namespace android
// ---------------------------------------------------------------------------

#endif // _LIBS_UTILS_MUTEX_H


text:
//
//  DDCreateGroupAPI.m
//  Duoduo
//
//  Created by 独嘉 on 14-5-8.
//  Copyright (c) 2015年 MoguIM All rights reserved.
//

#import "DDCreateGroupAPI.h"
#import "DDTcpProtocolHeader.h"
#import "MTTGroupEntity.h"
#import "MTTUserEntity.h"
#import "IMGroup.pb.h"

@implementation DDCreateGroupAPI
/**
 *  请求超时时间
 *
 *  @return 超时时间
 */
- (int)requestTimeOutTimeInterval
{
    return TimeOutTimeInterval;
}

/**
 *  请求的serviceID
 *
 *  @return 对应的serviceID
 */
- (int)requestServiceID
{
    return SID_GROUP;
}

/**
 *  请求返回的serviceID
 *
 *  @return 对应的serviceID
 */
- (int)responseServiceID
{
    return SID_GROUP;
}

/**
 *  请求的commendID
 *
 *  @return 对应的commendID
 */
- (int)requestCommendID
{
    return IM_GROUP_CREATE_REQ;
}

/**
 *  请求返回的commendID
 *
 *  @return 对应的commendID
 */
- (int)responseCommendID
{
    return IM_GROUP_CREATE_RES;
}

/**
 *  解析数据的block
 *
 *  @return 解析数据的block
 */
- (Analysis)analysisReturnData
{
    Analysis analysis = (id)^(NSData* data)
    {
        IMGroupCreateRsp *rsp = [IMGroupCreateRsp parseFromData:data];
        uint32_t result = rsp.resultCode;
        MTTGroupEntity* group = nil;
        if (result != 0)
        {
            return group;
        }
        else
        {
            NSString *groupId = [MTTGroupEntity pbGroupIdToLocalID:rsp.groupId];
            NSString *groupName = rsp.groupName;
            uint32_t userCnt =[[rsp userIdList] count];
            group = [[MTTGroupEntity alloc] init];
            group.objID = groupId;
            group.name = groupName;
            group.groupUserIds = [[NSMutableArray alloc] init];

            for (uint32_t i = 0; i < userCnt; i++) {
                NSString* userId = [MTTUserEntity pbUserIdToLocalID:[[rsp userIdList][i] integerValue]];
                [group.groupUserIds addObject:userId];
                [group addFixOrderGroupUserIDS:userId];
            }

            return group;
        }
    };
    return analysis;
}

/**
 *  打包数据的block
 *
 *  @return 打包数据的block
 */
- (Package)packageRequestObject
{
    Package package = (id)^(id object,uint16_t seqNo)
    {
        NSArray* array = (NSArray*)object;
        NSString* groupName = array[0];
        NSString* groupAvatar = array[1];
        NSArray* groupUserList = array[2];

        IMGroupCreateReqBuilder *req = [IMGroupCreateReq builder];
        [req setUserId:0];
        [req setGroupName:groupName];
        [req setGroupAvatar:groupAvatar];
        [req setGroupType:GroupTypeGroupTypeTmp];
        NSMutableArray *originalID = [NSMutableArray new];
        for (NSString *localID in groupUserList) {
            [originalID addObject:@([MTTUtil changeIDToOriginal:localID])];
        }
        [req setMemberIdListArray:originalID];
        DDDataOutputStream *dataout = [[DDDataOutputStream alloc] init];
        [dataout writeInt:0];
        [dataout writeTcpProtocolHeader:SID_GROUP
                                    cId:IM_GROUP_CREATE_REQ
                                  seqNo:seqNo];
        [dataout directWriteBytes:[req build].data];
        [dataout writeDataCount];
        return [dataout toByteArray];
    };
    return package;
}
@end


text:
<?php

use ActiveRecord\Column;
use ActiveRecord\DateTime;
use ActiveRecord\DatabaseException;

class ColumnTest extends SnakeCase_PHPUnit_Framework_TestCase
{
	public function set_up()
	{
		$this->column = new Column();
		try {
			$this->conn = ActiveRecord\ConnectionManager::get_connection(ActiveRecord\Config::instance()->get_default_connection());
		} catch (DatabaseException $e) {
			$this->mark_test_skipped('failed to connect using default connection. '.$e->getMessage());
		}
	}

	public function assert_mapped_type($type, $raw_type)
	{
		$this->column->raw_type = $raw_type;
		$this->assert_equals($type,$this->column->map_raw_type());
	}

	public function assert_cast($type, $casted_value, $original_value)
	{
		$this->column->type = $type;
		$value = $this->column->cast($original_value,$this->conn);

		if ($original_value != null && ($type == Column::DATETIME || $type == Column::DATE))
			$this->assert_true($value instanceof DateTime);
		else
			$this->assert_same($casted_value,$value);
	}

	public function test_map_raw_type_dates()
	{
		$this->assert_mapped_type(Column::DATETIME,'datetime');
		$this->assert_mapped_type(Column::DATE,'date');
	}

	public function test_map_raw_type_integers()
	{
		$this->assert_mapped_type(Column::INTEGER,'integer');
		$this->assert_mapped_type(Column::INTEGER,'int');
		$this->assert_mapped_type(Column::INTEGER,'tinyint');
		$this->assert_mapped_type(Column::INTEGER,'smallint');
		$this->assert_mapped_type(Column::INTEGER,'mediumint');
		$this->assert_mapped_type(Column::INTEGER,'bigint');
	}

	public function test_map_raw_type_decimals()
	{
		$this->assert_mapped_type(Column::DECIMAL,'float');
		$this->assert_mapped_type(Column::DECIMAL,'double');
		$this->assert_mapped_type(Column::DECIMAL,'numeric');
		$this->assert_mapped_type(Column::DECIMAL,'dec');
	}

	public function test_map_raw_type_strings()
	{
		$this->assert_mapped_type(Column::STRING,'string');
		$this->assert_mapped_type(Column::STRING,'varchar');
		$this->assert_mapped_type(Column::STRING,'text');
	}

	public function test_map_raw_type_default_to_string()
	{
		$this->assert_mapped_type(Column::STRING,'bajdslfjasklfjlksfd');
	}

	public function test_map_raw_type_changes_integer_to_int()
	{
		$this->column->raw_type = 'integer';
		$this->column->map_raw_type();
		$this->assert_equals('int',$this->column->raw_type);
	}

	public function test_cast()
	{
		$datetime = new DateTime('2001-01-01');
		$this->assert_cast(Column::INTEGER,1,'1');
		$this->assert_cast(Column::INTEGER,1,'1.5');
		$this->assert_cast(Column::DECIMAL,1.5,'1.5');
		$this->assert_cast(Column::DATETIME,$datetime,'2001-01-01');
		$this->assert_cast(Column::DATE,$datetime,'2001-01-01');
		$this->assert_cast(Column::DATE,$datetime,$datetime);
		$this->assert_cast(Column::STRING,'bubble tea','bubble tea');
		$this->assert_cast(Column::INTEGER,4294967295,'4294967295');
		$this->assert_cast(Column::INTEGER,'18446744073709551615','18446744073709551615');

		// 32 bit
		if (PHP_INT_SIZE === 4)
			$this->assert_cast(Column::INTEGER,'2147483648',(((float) PHP_INT_MAX) + 1));
		// 64 bit
		elseif (PHP_INT_SIZE === 8)
			$this->assert_cast(Column::INTEGER,'9223372036854775808',(((float) PHP_INT_MAX) + 1));
	}

	public function test_cast_leave_null_alone()
	{
		$types = array(
			Column::STRING,
			Column::INTEGER,
			Column::DECIMAL,
			Column::DATETIME,
			Column::DATE);

		foreach ($types as $type) {
			$this->assert_cast($type,null,null);
		}
	}

	public function test_empty_and_null_date_strings_should_return_null()
	{
		$column = new Column();
		$column->type = Column::DATE;
		$this->assert_equals(null,$column->cast(null,$this->conn));
		$this->assert_equals(null,$column->cast('',$this->conn));
	}

	public function test_empty_and_null_datetime_strings_should_return_null()
	{
		$column = new Column();
		$column->type = Column::DATETIME;
		$this->assert_equals(null,$column->cast(null,$this->conn));
		$this->assert_equals(null,$column->cast('',$this->conn));
	}
}
?>


text:
TenzingCore
===========

Some utilities for iOS. See [Wiki](https://github.com/endSly/TenzingCore/wiki) for more info.