text:
package org.apache.spark.deploy.rest

import java.io.DataOutputStream
import java.net.{HttpURLConnection, URL}
import java.nio.charset.StandardCharsets
import javax.servlet.http.HttpServletResponse

import scala.collection.mutable

import org.json4s.JsonAST._
import org.json4s.jackson.JsonMethods._
import org.scalatest.BeforeAndAfterEach

import org.apache.spark._
import org.apache.spark.deploy.{SparkSubmit, SparkSubmitArguments}
import org.apache.spark.deploy.DeployMessages._
import org.apache.spark.deploy.master.DriverState._
import org.apache.spark.rpc._
import org.apache.spark.util.Utils

/**
 * Tests for the REST application submission protocol used in standalone cluster mode.
 */
class StandaloneRestSubmitSuite extends SparkFunSuite with BeforeAndAfterEach {
  private var rpcEnv: Option[RpcEnv] = None
  private var server: Option[RestSubmissionServer] = None

  override def afterEach() {
    try {
      rpcEnv.foreach(_.shutdown())
      server.foreach(_.stop())
    } finally {
      super.afterEach()
    }
  }

  test("construct submit request") {
    val appArgs = Array("one", "two", "three")
    val sparkProperties = Map("spark.app.name" -> "pi")
    val environmentVariables = Map("SPARK_ONE" -> "UN", "SPARK_TWO" -> "DEUX")
    val request = new RestSubmissionClient("spark://host:port").constructSubmitRequest(
      "my-app-resource", "my-main-class", appArgs, sparkProperties, environmentVariables)
    assert(request.action === Utils.getFormattedClassName(request))
    assert(request.clientSparkVersion === SPARK_VERSION)
    assert(request.appResource === "my-app-resource")
    assert(request.mainClass === "my-main-class")
    assert(request.appArgs === appArgs)
    assert(request.sparkProperties === sparkProperties)
    assert(request.environmentVariables === environmentVariables)
  }

  test("create submission") {
    val submittedDriverId = "my-driver-id"
    val submitMessage = "your driver is submitted"
    val masterUrl = startDummyServer(submitId = submittedDriverId, submitMessage = submitMessage)
    val appArgs = Array("one", "two", "four")
    val request = constructSubmitRequest(masterUrl, appArgs)
    assert(request.appArgs === appArgs)
    assert(request.sparkProperties("spark.master") === masterUrl)
    val response = new RestSubmissionClient(masterUrl).createSubmission(request)
    val submitResponse = getSubmitResponse(response)
    assert(submitResponse.action === Utils.getFormattedClassName(submitResponse))
    assert(submitResponse.serverSparkVersion === SPARK_VERSION)
    assert(submitResponse.message === submitMessage)
    assert(submitResponse.submissionId === submittedDriverId)
    assert(submitResponse.success)
  }

  test("create submission from main method") {
    val submittedDriverId = "your-driver-id"
    val submitMessage = "my driver is submitted"
    val masterUrl = startDummyServer(submitId = submittedDriverId, submitMessage = submitMessage)
    val conf = new SparkConf(loadDefaults = false)
    conf.set("spark.master", masterUrl)
    conf.set("spark.app.name", "dreamer")
    val appArgs = Array("one", "two", "six")
    // main method calls this
    val response = RestSubmissionClient.run("app-resource", "main-class", appArgs, conf)
    val submitResponse = getSubmitResponse(response)
    assert(submitResponse.action === Utils.getFormattedClassName(submitResponse))
    assert(submitResponse.serverSparkVersion === SPARK_VERSION)
    assert(submitResponse.message === submitMessage)
    assert(submitResponse.submissionId === submittedDriverId)
    assert(submitResponse.success)
  }

  test("kill submission") {
    val submissionId = "my-lyft-driver"
    val killMessage = "your driver is killed"
    val masterUrl = startDummyServer(killMessage = killMessage)
    val response = new RestSubmissionClient(masterUrl).killSubmission(submissionId)
    val killResponse = getKillResponse(response)
    assert(killResponse.action === Utils.getFormattedClassName(killResponse))
    assert(killResponse.serverSparkVersion === SPARK_VERSION)
    assert(killResponse.message === killMessage)
    assert(killResponse.submissionId === submissionId)
    assert(killResponse.success)
  }

  test("request submission status") {
    val submissionId = "my-uber-driver"
    val submissionState = KILLED
    val submissionException = new Exception("there was an irresponsible mix of alcohol and cars")
    val masterUrl = startDummyServer(state = submissionState, exception = Some(submissionException))
    val response = new RestSubmissionClient(masterUrl).requestSubmissionStatus(submissionId)
    val statusResponse = getStatusResponse(response)
    assert(statusResponse.action === Utils.getFormattedClassName(statusResponse))
    assert(statusResponse.serverSparkVersion === SPARK_VERSION)
    assert(statusResponse.message.contains(submissionException.getMessage))
    assert(statusResponse.submissionId === submissionId)
    assert(statusResponse.driverState === submissionState.toString)
    assert(statusResponse.success)
  }

  test("create then kill") {
    val masterUrl = startSmartServer()
    val request = constructSubmitRequest(masterUrl)
    val client = new RestSubmissionClient(masterUrl)
    val response1 = client.createSubmission(request)
    val submitResponse = getSubmitResponse(response1)
    assert(submitResponse.success)
    assert(submitResponse.submissionId != null)
    // kill submission that was just created
    val submissionId = submitResponse.submissionId
    val response2 = client.killSubmission(submissionId)
    val killResponse = getKillResponse(response2)
    assert(killResponse.success)
    assert(killResponse.submissionId === submissionId)
  }

  test("create then request status") {
    val masterUrl = startSmartServer()
    val request = constructSubmitRequest(masterUrl)
    val client = new RestSubmissionClient(masterUrl)
    val response1 = client.createSubmission(request)
    val submitResponse = getSubmitResponse(response1)
    assert(submitResponse.success)
    assert(submitResponse.submissionId != null)
    // request status of submission that was just created
    val submissionId = submitResponse.submissionId
    val response2 = client.requestSubmissionStatus(submissionId)
    val statusResponse = getStatusResponse(response2)
    assert(statusResponse.success)
    assert(statusResponse.submissionId === submissionId)
    assert(statusResponse.driverState === RUNNING.toString)
  }

  test("create then kill then request status") {
    val masterUrl = startSmartServer()
    val request = constructSubmitRequest(masterUrl)
    val client = new RestSubmissionClient(masterUrl)
    val response1 = client.createSubmission(request)
    val response2 = client.createSubmission(request)
    val submitResponse1 = getSubmitResponse(response1)
    val submitResponse2 = getSubmitResponse(response2)
    assert(submitResponse1.success)
    assert(submitResponse2.success)
    assert(submitResponse1.submissionId != null)
    assert(submitResponse2.submissionId != null)
    val submissionId1 = submitResponse1.submissionId
    val submissionId2 = submitResponse2.submissionId
    // kill only submission 1, but not submission 2
    val response3 = client.killSubmission(submissionId1)
    val killResponse = getKillResponse(response3)
    assert(killResponse.success)
    assert(killResponse.submissionId === submissionId1)
    // request status for both submissions: 1 should be KILLED but 2 should be RUNNING still
    val response4 = client.requestSubmissionStatus(submissionId1)
    val response5 = client.requestSubmissionStatus(submissionId2)
    val statusResponse1 = getStatusResponse(response4)
    val statusResponse2 = getStatusResponse(response5)
    assert(statusResponse1.submissionId === submissionId1)
    assert(statusResponse2.submissionId === submissionId2)
    assert(statusResponse1.driverState === KILLED.toString)
    assert(statusResponse2.driverState === RUNNING.toString)
  }

  test("kill or request status before create") {
    val masterUrl = startSmartServer()
    val doesNotExist = "does-not-exist"
    val client = new RestSubmissionClient(masterUrl)
    // kill a non-existent submission
    val response1 = client.killSubmission(doesNotExist)
    val killResponse = getKillResponse(response1)
    assert(!killResponse.success)
    assert(killResponse.submissionId === doesNotExist)
    // request status for a non-existent submission
    val response2 = client.requestSubmissionStatus(doesNotExist)
    val statusResponse = getStatusResponse(response2)
    assert(!statusResponse.success)
    assert(statusResponse.submissionId === doesNotExist)
  }

  /* ---------------------------------------- *
   |     Aberrant client / server behavior    |
   * ---------------------------------------- */

  test("good request paths") {
    val masterUrl = startSmartServer()
    val httpUrl = masterUrl.replace("spark://", "http://")
    val v = RestSubmissionServer.PROTOCOL_VERSION
    val json = constructSubmitRequest(masterUrl).toJson
    val submitRequestPath = s"$httpUrl/$v/submissions/create"
    val killRequestPath = s"$httpUrl/$v/submissions/kill"
    val statusRequestPath = s"$httpUrl/$v/submissions/status"
    val (response1, code1) = sendHttpRequestWithResponse(submitRequestPath, "POST", json)
    val (response2, code2) = sendHttpRequestWithResponse(s"$killRequestPath/anything", "POST")
    val (response3, code3) = sendHttpRequestWithResponse(s"$killRequestPath/any/thing", "POST")
    val (response4, code4) = sendHttpRequestWithResponse(s"$statusRequestPath/anything", "GET")
    val (response5, code5) = sendHttpRequestWithResponse(s"$statusRequestPath/any/thing", "GET")
    // these should all succeed and the responses should be of the correct types
    getSubmitResponse(response1)
    val killResponse1 = getKillResponse(response2)
    val killResponse2 = getKillResponse(response3)
    val statusResponse1 = getStatusResponse(response4)
    val statusResponse2 = getStatusResponse(response5)
    assert(killResponse1.submissionId === "anything")
    assert(killResponse2.submissionId === "any")
    assert(statusResponse1.submissionId === "anything")
    assert(statusResponse2.submissionId === "any")
    assert(code1 === HttpServletResponse.SC_OK)
    assert(code2 === HttpServletResponse.SC_OK)
    assert(code3 === HttpServletResponse.SC_OK)
    assert(code4 === HttpServletResponse.SC_OK)
    assert(code5 === HttpServletResponse.SC_OK)
  }

  test("good request paths, bad requests") {
    val masterUrl = startSmartServer()
    val httpUrl = masterUrl.replace("spark://", "http://")
    val v = RestSubmissionServer.PROTOCOL_VERSION
    val submitRequestPath = s"$httpUrl/$v/submissions/create"
    val killRequestPath = s"$httpUrl/$v/submissions/kill"
    val statusRequestPath = s"$httpUrl/$v/submissions/status"
    val goodJson = constructSubmitRequest(masterUrl).toJson
    val badJson1 = goodJson.replaceAll("action", "fraction") // invalid JSON
    val badJson2 = goodJson.substring(goodJson.size / 2) // malformed JSON
    val notJson = "\"hello, world\""
    val (response1, code1) = sendHttpRequestWithResponse(submitRequestPath, "POST") // missing JSON
    val (response2, code2) = sendHttpRequestWithResponse(submitRequestPath, "POST", badJson1)
    val (response3, code3) = sendHttpRequestWithResponse(submitRequestPath, "POST", badJson2)
    val (response4, code4) = sendHttpRequestWithResponse(killRequestPath, "POST") // missing ID
    val (response5, code5) = sendHttpRequestWithResponse(s"$killRequestPath/", "POST")
    val (response6, code6) = sendHttpRequestWithResponse(statusRequestPath, "GET") // missing ID
    val (response7, code7) = sendHttpRequestWithResponse(s"$statusRequestPath/", "GET")
    val (response8, code8) = sendHttpRequestWithResponse(submitRequestPath, "POST", notJson)
    // these should all fail as error responses
    getErrorResponse(response1)
    getErrorResponse(response2)
    getErrorResponse(response3)
    getErrorResponse(response4)
    getErrorResponse(response5)
    getErrorResponse(response6)
    getErrorResponse(response7)
    getErrorResponse(response8)
    assert(code1 === HttpServletResponse.SC_BAD_REQUEST)
    assert(code2 === HttpServletResponse.SC_BAD_REQUEST)
    assert(code3 === HttpServletResponse.SC_BAD_REQUEST)
    assert(code4 === HttpServletResponse.SC_BAD_REQUEST)
    assert(code5 === HttpServletResponse.SC_BAD_REQUEST)
    assert(code6 === HttpServletResponse.SC_BAD_REQUEST)
    assert(code7 === HttpServletResponse.SC_BAD_REQUEST)
    assert(code8 === HttpServletResponse.SC_BAD_REQUEST)
  }

  test("bad request paths") {
    val masterUrl = startSmartServer()
    val httpUrl = masterUrl.replace("spark://", "http://")
    val v = RestSubmissionServer.PROTOCOL_VERSION
    val (response1, code1) = sendHttpRequestWithResponse(httpUrl, "GET")
    val (response2, code2) = sendHttpRequestWithResponse(s"$httpUrl/", "GET")
    val (response3, code3) = sendHttpRequestWithResponse(s"$httpUrl/$v", "GET")
    val (response4, code4) = sendHttpRequestWithResponse(s"$httpUrl/$v/", "GET")
    val (response5, code5) = sendHttpRequestWithResponse(s"$httpUrl/$v/submissions", "GET")
    val (response6, code6) = sendHttpRequestWithResponse(s"$httpUrl/$v/submissions/", "GET")
    val (response7, code7) = sendHttpRequestWithResponse(s"$httpUrl/$v/submissions/bad", "GET")
    val (response8, code8) = sendHttpRequestWithResponse(s"$httpUrl/bad-version", "GET")
    assert(code1 === HttpServletResponse.SC_BAD_REQUEST)
    assert(code2 === HttpServletResponse.SC_BAD_REQUEST)
    assert(code3 === HttpServletResponse.SC_BAD_REQUEST)
    assert(code4 === HttpServletResponse.SC_BAD_REQUEST)
    assert(code5 === HttpServletResponse.SC_BAD_REQUEST)
    assert(code6 === HttpServletResponse.SC_BAD_REQUEST)
    assert(code7 === HttpServletResponse.SC_BAD_REQUEST)
    assert(code8 === RestSubmissionServer.SC_UNKNOWN_PROTOCOL_VERSION)
    // all responses should be error responses
    val errorResponse1 = getErrorResponse(response1)
    val errorResponse2 = getErrorResponse(response2)
    val errorResponse3 = getErrorResponse(response3)
    val errorResponse4 = getErrorResponse(response4)
    val errorResponse5 = getErrorResponse(response5)
    val errorResponse6 = getErrorResponse(response6)
    val errorResponse7 = getErrorResponse(response7)
    val errorResponse8 = getErrorResponse(response8)
    // only the incompatible version response should have server protocol version set
    assert(errorResponse1.highestProtocolVersion === null)
    assert(errorResponse2.highestProtocolVersion === null)
    assert(errorResponse3.highestProtocolVersion === null)
    assert(errorResponse4.highestProtocolVersion === null)
    assert(errorResponse5.highestProtocolVersion === null)
    assert(errorResponse6.highestProtocolVersion === null)
    assert(errorResponse7.highestProtocolVersion === null)
    assert(errorResponse8.highestProtocolVersion === RestSubmissionServer.PROTOCOL_VERSION)
  }

  test("server returns unknown fields") {
    val masterUrl = startSmartServer()
    val httpUrl = masterUrl.replace("spark://", "http://")
    val v = RestSubmissionServer.PROTOCOL_VERSION
    val submitRequestPath = s"$httpUrl/$v/submissions/create"
    val oldJson = constructSubmitRequest(masterUrl).toJson
    val oldFields = parse(oldJson).asInstanceOf[JObject].obj
    val newFields = oldFields ++ Seq(
      JField("tomato", JString("not-a-fruit")),
      JField("potato", JString("not-po-tah-to"))
    )
    val newJson = pretty(render(JObject(newFields)))
    // send two requests, one with the unknown fields and the other without
    val (response1, code1) = sendHttpRequestWithResponse(submitRequestPath, "POST", oldJson)
    val (response2, code2) = sendHttpRequestWithResponse(submitRequestPath, "POST", newJson)
    val submitResponse1 = getSubmitResponse(response1)
    val submitResponse2 = getSubmitResponse(response2)
    assert(code1 === HttpServletResponse.SC_OK)
    assert(code2 === HttpServletResponse.SC_OK)
    // only the response to the modified request should have unknown fields set
    assert(submitResponse1.unknownFields === null)
    assert(submitResponse2.unknownFields === Array("tomato", "potato"))
  }

  test("client handles faulty server") {
    val masterUrl = startFaultyServer()
    val client = new RestSubmissionClient(masterUrl)
    val httpUrl = masterUrl.replace("spark://", "http://")
    val v = RestSubmissionServer.PROTOCOL_VERSION
    val submitRequestPath = s"$httpUrl/$v/submissions/create"
    val killRequestPath = s"$httpUrl/$v/submissions/kill/anything"
    val statusRequestPath = s"$httpUrl/$v/submissions/status/anything"
    val json = constructSubmitRequest(masterUrl).toJson
    // server returns malformed response unwittingly
    // client should throw an appropriate exception to indicate server failure
    val conn1 = sendHttpRequest(submitRequestPath, "POST", json)
    intercept[SubmitRestProtocolException] { client.readResponse(conn1) }
    // server attempts to send invalid response, but fails internally on validation
    // client should receive an error response as server is able to recover
    val conn2 = sendHttpRequest(killRequestPath, "POST")
    val response2 = client.readResponse(conn2)
    getErrorResponse(response2)
    assert(conn2.getResponseCode === HttpServletResponse.SC_INTERNAL_SERVER_ERROR)
    // server explodes internally beyond recovery
    // client should throw an appropriate exception to indicate server failure
    val conn3 = sendHttpRequest(statusRequestPath, "GET")
    intercept[SubmitRestProtocolException] { client.readResponse(conn3) } // empty response
    assert(conn3.getResponseCode === HttpServletResponse.SC_INTERNAL_SERVER_ERROR)
  }

  test("client does not send 'SPARK_ENV_LOADED' env var by default") {
    val environmentVariables = Map("SPARK_VAR" -> "1", "SPARK_ENV_LOADED" -> "1")
    val filteredVariables = RestSubmissionClient.filterSystemEnvironment(environmentVariables)
    assert(filteredVariables == Map("SPARK_VAR" -> "1"))
  }

  test("client includes mesos env vars") {
    val environmentVariables = Map("SPARK_VAR" -> "1", "MESOS_VAR" -> "1", "OTHER_VAR" -> "1")
    val filteredVariables = RestSubmissionClient.filterSystemEnvironment(environmentVariables)
    assert(filteredVariables == Map("SPARK_VAR" -> "1", "MESOS_VAR" -> "1"))
  }

  /* --------------------- *
   |     Helper methods    |
   * --------------------- */

  /** Start a dummy server that responds to requests using the specified parameters. */
  private def startDummyServer(
      submitId: String = "fake-driver-id",
      submitMessage: String = "driver is submitted",
      killMessage: String = "driver is killed",
      state: DriverState = FINISHED,
      exception: Option[Exception] = None): String = {
    startServer(new DummyMaster(_, submitId, submitMessage, killMessage, state, exception))
  }

  /** Start a smarter dummy server that keeps track of submitted driver states. */
  private def startSmartServer(): String = {
    startServer(new SmarterMaster(_))
  }

  /** Start a dummy server that is faulty in many ways... */
  private def startFaultyServer(): String = {
    startServer(new DummyMaster(_), faulty = true)
  }

  /**
   * Start a [[StandaloneRestServer]] that communicates with the given endpoint.
   * If `faulty` is true, start a [[FaultyStandaloneRestServer]] instead.
   * Return the master URL that corresponds to the address of this server.
   */
  private def startServer(
      makeFakeMaster: RpcEnv => RpcEndpoint, faulty: Boolean = false): String = {
    val name = "test-standalone-rest-protocol"
    val conf = new SparkConf
    val localhost = Utils.localHostName()
    val securityManager = new SecurityManager(conf)
    val _rpcEnv = RpcEnv.create(name, localhost, 0, conf, securityManager)
    val fakeMasterRef = _rpcEnv.setupEndpoint("fake-master", makeFakeMaster(_rpcEnv))
    val _server =
      if (faulty) {
        new FaultyStandaloneRestServer(localhost, 0, conf, fakeMasterRef, "spark://fake:7077")
      } else {
        new StandaloneRestServer(localhost, 0, conf, fakeMasterRef, "spark://fake:7077")
      }
    val port = _server.start()
    // set these to clean them up after every test
    rpcEnv = Some(_rpcEnv)
    server = Some(_server)
    s"spark://$localhost:$port"
  }

  /** Create a submit request with real parameters using Spark submit. */
  private def constructSubmitRequest(
      masterUrl: String,
      appArgs: Array[String] = Array.empty): CreateSubmissionRequest = {
    val mainClass = "main-class-not-used"
    val mainJar = "dummy-jar-not-used.jar"
    val commandLineArgs = Array(
      "--deploy-mode", "cluster",
      "--master", masterUrl,
      "--name", mainClass,
      "--class", mainClass,
      mainJar) ++ appArgs
    val args = new SparkSubmitArguments(commandLineArgs)
    val (_, _, sparkProperties, _, _, _) = SparkSubmit.prepareSubmitEnvironment(args)
    new RestSubmissionClient("spark://host:port").constructSubmitRequest(
      mainJar, mainClass, appArgs, sparkProperties.toMap, Map.empty)
  }

  /** Return the response as a submit response, or fail with error otherwise. */
  private def getSubmitResponse(response: SubmitRestProtocolResponse): CreateSubmissionResponse = {
    response match {
      case s: CreateSubmissionResponse => s
      case e: ErrorResponse => fail(s"Server returned error: ${e.message}")
      case r => fail(s"Expected submit response. Actual: ${r.toJson}")
    }
  }

  /** Return the response as a kill response, or fail with error otherwise. */
  private def getKillResponse(response: SubmitRestProtocolResponse): KillSubmissionResponse = {
    response match {
      case k: KillSubmissionResponse => k
      case e: ErrorResponse => fail(s"Server returned error: ${e.message}")
      case r => fail(s"Expected kill response. Actual: ${r.toJson}")
    }
  }

  /** Return the response as a status response, or fail with error otherwise. */
  private def getStatusResponse(response: SubmitRestProtocolResponse): SubmissionStatusResponse = {
    response match {
      case s: SubmissionStatusResponse => s
      case e: ErrorResponse => fail(s"Server returned error: ${e.message}")
      case r => fail(s"Expected status response. Actual: ${r.toJson}")
    }
  }

  /** Return the response as an error response, or fail if the response was not an error. */
  private def getErrorResponse(response: SubmitRestProtocolResponse): ErrorResponse = {
    response match {
      case e: ErrorResponse => e
      case r => fail(s"Expected error response. Actual: ${r.toJson}")
    }
  }

  /**
   * Send an HTTP request to the given URL using the method and the body specified.
   * Return the connection object.
   */
  private def sendHttpRequest(
      url: String,
      method: String,
      body: String = ""): HttpURLConnection = {
    val conn = new URL(url).openConnection().asInstanceOf[HttpURLConnection]
    conn.setRequestMethod(method)
    if (body.nonEmpty) {
      conn.setDoOutput(true)
      val out = new DataOutputStream(conn.getOutputStream)
      out.write(body.getBytes(StandardCharsets.UTF_8))
      out.close()
    }
    conn
  }

  /**
   * Send an HTTP request to the given URL using the method and the body specified.
   * Return a 2-tuple of the response message from the server and the response code.
   */
  private def sendHttpRequestWithResponse(
      url: String,
      method: String,
      body: String = ""): (SubmitRestProtocolResponse, Int) = {
    val conn = sendHttpRequest(url, method, body)
    (new RestSubmissionClient("spark://host:port").readResponse(conn), conn.getResponseCode)
  }
}

/**
 * A mock standalone Master that responds with dummy messages.
 * In all responses, the success parameter is always true.
 */
private class DummyMaster(
    override val rpcEnv: RpcEnv,
    submitId: String = "fake-driver-id",
    submitMessage: String = "submitted",
    killMessage: String = "killed",
    state: DriverState = FINISHED,
    exception: Option[Exception] = None)
  extends RpcEndpoint {

  override def receiveAndReply(context: RpcCallContext): PartialFunction[Any, Unit] = {
    case RequestSubmitDriver(driverDesc) =>
      context.reply(SubmitDriverResponse(self, success = true, Some(submitId), submitMessage))
    case RequestKillDriver(driverId) =>
      context.reply(KillDriverResponse(self, driverId, success = true, killMessage))
    case RequestDriverStatus(driverId) =>
      context.reply(DriverStatusResponse(found = true, Some(state), None, None, exception))
  }
}

/**
 * A mock standalone Master that keeps track of drivers that have been submitted.
 *
 * If a driver is submitted, its state is immediately set to RUNNING.
 * If an existing driver is killed, its state is immediately set to KILLED.
 * If an existing driver's status is requested, its state is returned in the response.
 * Submits are always successful while kills and status requests are successful only
 * if the driver was submitted in the past.
 */
private class SmarterMaster(override val rpcEnv: RpcEnv) extends ThreadSafeRpcEndpoint {
  private var counter: Int = 0
  private val submittedDrivers = new mutable.HashMap[String, DriverState]

  override def receiveAndReply(context: RpcCallContext): PartialFunction[Any, Unit] = {
    case RequestSubmitDriver(driverDesc) =>
      val driverId = s"driver-$counter"
      submittedDrivers(driverId) = RUNNING
      counter += 1
      context.reply(SubmitDriverResponse(self, success = true, Some(driverId), "submitted"))

    case RequestKillDriver(driverId) =>
      val success = submittedDrivers.contains(driverId)
      if (success) {
        submittedDrivers(driverId) = KILLED
      }
      context.reply(KillDriverResponse(self, driverId, success, "killed"))

    case RequestDriverStatus(driverId) =>
      val found = submittedDrivers.contains(driverId)
      val state = submittedDrivers.get(driverId)
      context.reply(DriverStatusResponse(found, state, None, None, None))
  }
}

/**
 * A [[StandaloneRestServer]] that is faulty in many ways.
 *
 * When handling a submit request, the server returns a malformed JSON.
 * When handling a kill request, the server returns an invalid JSON.
 * When handling a status request, the server throws an internal exception.
 * The purpose of this class is to test that client handles these cases gracefully.
 */
private class FaultyStandaloneRestServer(
    host: String,
    requestedPort: Int,
    masterConf: SparkConf,
    masterEndpoint: RpcEndpointRef,
    masterUrl: String)
  extends RestSubmissionServer(host, requestedPort, masterConf) {

  protected override val submitRequestServlet = new MalformedSubmitServlet
  protected override val killRequestServlet = new InvalidKillServlet
  protected override val statusRequestServlet = new ExplodingStatusServlet

  /** A faulty servlet that produces malformed responses. */
  class MalformedSubmitServlet
    extends StandaloneSubmitRequestServlet(masterEndpoint, masterUrl, masterConf) {
    protected override def sendResponse(
        responseMessage: SubmitRestProtocolResponse,
        responseServlet: HttpServletResponse): Unit = {
      val badJson = responseMessage.toJson.drop(10).dropRight(20)
      responseServlet.getWriter.write(badJson)
    }
  }

  /** A faulty servlet that produces invalid responses. */
  class InvalidKillServlet extends StandaloneKillRequestServlet(masterEndpoint, masterConf) {
    protected override def handleKill(submissionId: String): KillSubmissionResponse = {
      val k = super.handleKill(submissionId)
      k.submissionId = null
      k
    }
  }

  /** A faulty status servlet that explodes. */
  class ExplodingStatusServlet extends StandaloneStatusRequestServlet(masterEndpoint, masterConf) {
    private def explode: Int = 1 / 0
    protected override def handleStatus(submissionId: String): SubmissionStatusResponse = {
      val s = super.handleStatus(submissionId)
      s.workerId = explode.toString
      s
    }
  }
}

redpajama_set_name: RedPajamaGithub


Sara Magenheimer (b. Philadelphia) has widely exhibited, performed and screened her work. Recent exhibitions include White Columns, Foxy Production, Document, Chapter NY, Interstate Projects, 247365, and Cleopatra's. From 2004–10 Magenheimer performed in bands, touring extensively and releasing five records. She was the recipient of a 2014 Rema Hort Mann Foundation Grant, 2015 Artadia Grant, and winner of the Prix De Varti at the 2015 Ann Arbor Film Festival. Recent exhibitions include Art In General's New Commission in Riga, Latvia and a solo exhibition at The Kitchen in New York.
Artforum: "The View from the Window"


The beautiful kahikatea, kauri, tōtara, kōwhai, rewarewa, rimu, extensive range of pseudopanax and pittosporums, mānuka, kānuka, hebes along with ground covers, flaxes, grasses and the magnificent pōhutakawa.
So many to choose from whether planting back out in nature or creating native areas in our gardens.
In truth natives are not so easy to propagate, but our professional team of propagators through the process of seed stratification, replicate nature's rules for breaking seed dormancy in order to promote germination.
For a few of our staff natives are a special passion, actively involved in native bush restoration projects.


text:
Q: SQL Server Delete Lock issue I have a SQL Server database where I am deleting rows from three tables A,B,C in batches with some conditions through a SQL script scheduled in a SQL job. The job runs for 2 hours as the tables have a large amount of data. While the job is running, my front end application is not accessible (giving timeout error) since the application inserts and updates data in these same tables A,B,C.
Is it possible for the front end application to run in parallel without any issues while the SQL script is running? I have checked for the locks on the table and SQL Server is acquiring page locks. Can Read Committed Snapshot or Snapshot isolation levels or converting page locks to row locks help here. Need advice.

A: Split the operation in two phases. In the first phase, collect the primary keys of rows to delete:
create table #TempList (ID int);

insert  #TempList
select  ID
from    YourTable

In the second phase, use a loop to delete those rows in small batches:
while 1=1
    begin
    delete  top (1000)
    from    YourTable
    where   ID in (select ID from #TempList)

    if @@rowcount = 0
        break
    end

The smaller batches will allow your front end applications to continue in between them.

A: I suspect that SQL Server at some point escalates to table lock, and this means that the table is inaccessible, both for reading and updating.
To optimize locking and concurrency when dealing with large deletes, use batches. Start with 5000 rows at the time (to prevent lock escalation) and monitor how it behaves and whether it needs further tuning up or down. 5000 is a "magic number", but it's low enough number that lock manager doesn't consider escalating to table lock, and large enough for the performance.
Whether timeouts will happen or not depends on other factors as well, but this will surely reduce if not elliminate alltogether. If the timeout happen on read operations, you should be able to get rid of them. Another approach, of course, is to increase the command timeout value on client.
Snapshot (optimistic) isolation is an option as well, READ COMMITTED SNAPSHOT more precisely, but it won't help with updates from other sessions. Also, beware of version store (in tempdb) growth. Best if you combine it with the proposed batch approach to keep the transactions small.
Also, switch to bulk-logged recovery for the duration of delete if the database is in full recovery normally. But switch back as soon as it finishes, and make a backup.
Almost forgot -- if it's Enterprise edition of SQL Server, partition your table; then you can just switch the partition out, it's almost momentarilly and the clients will never notice it.

redpajama_set_name: RedPajamaStackExchange


I'm sick to death of self proclaimed business gurus attempting to crystallise their "expert" opinion of themselves, capping off their mediocre careers by writing a book. I find it an offensive waste of my time when they waffle on for 13 hours about their opinions without any real authority on their subject, but hey, if you're driving for hours on end you might as well listen to something semi-educational right?
This book is DIFFERENT. Oren Klaff has put together a fantastic book that will change the way you perceive social situations and help you pitch your ideas successfully. He narrates it himself and keeps you 100% captivated.
1. Listen to it at regular 1x speed because Oren doesn't waffle or talk to you slowly like you're retarded. Oren respects your time.
2. Pause at times to take notes or contemplate some awesome new concept. You may even tell your spouse how interesting the book you're listening to is.
3. Listen to it over and over again, because there is so much useful information that it may take a while to put it all into practise.
There are tonnes of books with 5 star reviews on Audible that don't deserve them, this book stands out as one that really deserves a 5 star review and it completely worthy of your time.
The benchmark guide to marketing and PR, updated with the latest social media and marketing trends, tools, and real-world examples of success The New Rules of Marketing & PR, 4th Edition is the pioneering guide to the future of marketing, an international best seller with more than 300,000 copies sold in over 25 languages. It offers a step-by-step action plan for harnessing the power of modern marketing and PR to communicate with buyers directly, raise visibility, and increase sales.
I read this book on the advice that it would help me set up a social media strategy. It did cover a few social media platforms but omitted many of the newer big ones making me question the "new" in the title. Overall it didn't help me set up a strategy at all. I still don't know whether I should blog or tweet, put photos up on Pinterest or Facebook, what content should go on my website's news feed and can I cross post? Overall this book just offered a basic intro to some of the platforms and I'm no closer to having a solid social media strategy.
Six Disciplines Leadership Centers are dedicated to helping small businesses achieve and sustain success. Six Disciplines for Excellence provides six fundamental business principles that are specifically designed to help small businesses move beyond momentary success to attain enduring excellence. This audiobook is not for those who are looking for a quick fix. Six Disciplines for Excellence is a long-term fitness program, not a fad diet.
Not suitable as an audiobook.
This book reads like a textbook rather than a story. It is broken down into numbered lists and sub lists e.g. "Section 4a" and I found it very hard to follow. The book also makes excessive use of the word etcetera to the point that you can tell when they're going to use it and speak along in unison.
The content feels very dated. They make a point that you "should" start thinking about getting on this new thing called The Internet and they have to spell out what IT stands for.
Worst of all the book culminates in a sales pitch about how good the (paid for) six disciplines software is, yet it sounds like a sub-par version of Trello (which is free by the way).
I regret wasting my monthly audible credit on this book and I wish I could get my 4 hours back. I call shenanigans on all of the 5 star reviews this book has received.
This book is a bit of a let-down. You will learn a bit about Elon and his companies, but it will sap 13 hours of your life away and you won't be a better person for it.
You'd think there's so much you can learn from Elon Musk, but this author didn't really make much effort to gather any nuggets of insightful information, just snippets of hearsay from sources that have worked around Elon. Listening to the book feels kind of perverted because it's written in a way where the author digs up lots of dirt on him the way a trash magazine reports on the secret lives of celebrities.
The Author even gloats mildly about his tenacity in pestering Musk until he agreed to let him write the book.
Don't bother listening to this one.
An Astronaut's Guide to Life on Earth is an inspirational memoir of space exploration and hard-won wisdom, from an astronaut who has spent a lifetime making the impossible a reality. Colonel Chris Hadfield has spent decades training as an astronaut and has logged nearly 4,000 hours in space. During this time he has broken into a Space Station with a Swiss army knife, disposed of a live snake while piloting a plane, been temporarily blinded while clinging to the exterior of an orbiting spacecraft, and become a YouTube sensation with his performance of David Bowie's 'Space Oddity' in space.
Before reading this book I already had a lot of respect for Chris and I still do. He comes across as an extremely focussed, dedicated and successful man so I was eager to hear how he managed his relationship with his family.
While Chris pays every respect to his wife and children for their countless contributions and sacrifices, I found myself a little disappointed that he found no elegant way to fulfil his family obligations while striving to be the best he could be in his professional career. Sure, he did the best he could and made sure his family knew they were important to him - whenever he had the time. He planned thoughtful gestures well in advance of away missions to try and make up for his absence. In a nutshell, he could have quit at any time but his career always came first. Anything less and he would never have become an astronaut.
I still can't decide whether that's right or not. It's a lot to think about.
Simply the most endorsed start-up book ever written. Hailed by the academic community, New York Times best-selling authors, as well as established entrepreneurs. If you cannot afford to fail, this is the one book you want to listen to. Written by a 20-year serial entrepreneur, in a straightforward language with key takeaways and action steps, this book guides you through the rules to follow for starting a successful business.
I downloaded this book after hearing reviews that it was an essential read. Although I got a few good ideas from it, it's pretty basic and aimed at people in the beginning stages of a business.
My main gripe is that it's best suited to regular businesses like restaurants and not for lean tech startups. It preaches a lot of non-lean traditional advice that buries you in planning before you even begin.
There is a lot of great advice in this book but some of the tactics are manipulative and I feel that most people would see right through them. The main saving grace is that you must be genuine in caring about others, otherwise this book will be a waste of time.
The book gives a number of success stories then attributes their success with the fact that they started with "why". I'm dubious that their success is based on just this and not the myriad external factors and some luck, but overall I think there is a good argument that we should start out with why.
I am glad I read the book even if its logic isn't entirely sound.
It is a cold, hard fact of business life that most startups fail. Even many of those entrepreneurs who ultimately succeed have stories of personal challenges, unsuccessful companies, and difficulties along the way. The founders of TechStars, a mentorship-driven startup accelerator, have worked with entrepreneurs and companies over the past twenty-five years, and have seen a number of the same issues come up again and again.
This book is made up of short stories, each with an overarching lesson. Some of the stories are only relevant to US startups like incorporating in Delaware, but most of the info is universally useful.
This book taught me a way to understand how the mind thinks and reacts to situations. This model is very fitting of my own experiences and helps me understand the way people react when they get angry or just rubbed the wrong way. Although this is good information, it should have taken no longer than 30 minutes to explain. The remaining 8.5 hours of this book was tedious and redundant.


text:
How Beach made a statement edging Johnson in battle of two of state's best
Dennis Knight
Savannah basketball teams sent a message to the state of Georgia last year as all four boys teams from Region 3-3A reached the quarterfinals in the state playoffs.
Beach caught the attention of basketball fans as the Bulldogs, the No. 4 seed coming out of the region, went on the road to win three straight games en route to a berth in the Final Four, where they lost to eventual state champion Cross Creek -- a winner over Windsor Forest in the finals.
On Friday, Beach made a statement once again as the Bulldogs, who entered the game ranked No. 4 among Class 3A teams in the state by Sandysspiel.com, edged visiting Johnson, ranked No. 2, 64-57 in a game that came down to the final minute.
Beach improved to 11-1 and 3-0 in region play, while Johnson fell to 10-3 and 2-2 in a game that had everything the overflowing crowd in the Beach gymnasium could ask for.
Top Savannah basketball players to watchThe top high school boys basketball players to watch in the Greater Savannah area
Breakout boys hoopsters to watchHere's a list of boys basketball breakout players to watch in the Greater Savannah area
2022 All-Greater Savannah football teamHere's the 2022 All-Greater Savannah high school football team and Coach of the Year
Two of the city's biggest stars were in the mix as Beach senior Shamarrie Hugie showed off his skills with 24 points before fouling out with 4:54 left. Johnson senior Antonio Baker, who averages 23 points per game, didn't have a big scoring night with 13 points -- but he found others ways to impact the game with seven steals, three assists and three rebounds before fouling out with 1:01 left.
Beach coach Simon Heyward drew a technical foul when Hugie was whistled for his fifth and final foul with the game knotted at 52 with 4:54 left.
"I was keeping that tech in my back pocket to rile up the boys if we needed it," Heyward said with a laugh. "We knew Shamarrie going out was going to be a big loss, but our guys are battle tested and we're used to being in back-and-forth games that come down to the final seconds."
Sophomore guard Chandler Reid did an excellent job running the point all night, especially in the closing minutes. He finished with 18 points. Tyrese Bellamy was strong defensively and had seven points, with James Tisby adding six and James Leach scoring all six of his points in the second half -- including the final four points as Beach closed things out.
"Once Shamarrie went out, I had to think about just controlling the game and not making any turnovers," Reid said. "We just needed to do what we do best down the stretch with our defense. There was a lot trash talking about how they had a higher ranking than us, so we felt like we had to win this game. But we have a lot of respect for Johnson. They have a lot of skillful players. They pass the ball and play good defense too."
Hugie was proud of the way his teammates responded after he fouled out.
"I trusted my boys," he said. "I was mad when I went out, but it's not the first time we've been in that situation. We're experienced and trusted each other to close it out. Johnson is a really good team, but we stayed focused on our defense and that helped us win it."
Johnson junior guard Josh Quarterman, a transfer from Bethesda, has fit in perfectly with Coach Chuck Campbell's system and proven to be one of the top players in the city. He had another impressive performance Friday with 13 points, while emerging standout Favion Kirkwood, a sophomore guard, added 10 points and five rebounds.
The Atom Smashers (11-3, 2-2 in Region 3-3A) bounced back with a 56-51 win over Paxon (FL) in the Pre-MLK Classic in Jacksonville Saturday as Baker scored 21 points with eight rebounds with Kirkwood adding 20 points and seven boards and Malachi Robinson grabbing 18 rebounds.
Beach (11-2, 3-0 in Region 3-3A) lost Saturday in the same event in a 56-49 setback to SLAM Academy with Hugie scoring 23 for the Bulldogs, who are set to host Groves in a Region 3-3A matchup Tuesday, while Johnson hits the road for a region matchup at Long County.
Friday's game was a huge matchup in front of a sellout crowd in an early round battle of what promises to be an exciting race for the Region 3-3A title, and the ensuing run for a state crown, with Beach and Johnson favorites, along with a talented Calvary Day (10-3) squad, which has a region win over Johnson and is ranked No. 6 in Class 3A by Sandysspiel.com.
In the Class 3A state rankings released Sunday by Sandysspiel.com, Beach moved up to No. 2, with Johnson at No. 3 and Calvary Day sitting at No. 5. Sandy Creek (11-3) is the top-ranked team in Class 3A.
Heyward knows there will be more local battles along the way, including the Bulldogs game at Johnson set for February 3.
"Every time we play Johnson it's going to be a big game," Heyward said. "Coach Chuck (Campbell) and his team are going to bring their 'A' game, and we know it's going to come down to the last few minutes. We'll be ready for it."
Campbell was disappointed with his team's play down the stretch Friday night, but thinks it will help the Atom Smashers in the long run.
"Beach is a scrappy team, but once Shamarrie (Hugie) fouled out we should have slowed down the game and run our sets," Campbell said. "We played selfish tonight with too much 1-on-1 and every time we do that -- we lose. I think this is going to help us going forward. We're tough, we're going to lick our wounds and come back refocused."
Dennis Knight covers sports for the Savannah Morning News. Contact him at Dknight@savannahnow.com. Twitter: @DennisKnightSMN

redpajama_set_name: RedPajamaCommonCrawl


Bookmark this page and use it daily.
This page offers all the most important links you need to drive traffic and analyze your website performances.
I regrouped all the links and services I'm using myself daily to, first of all, organize myself. It's much easier to follow a strict routine when all the links are arranged by categories on one page. My Firefox bookmarks are crowded with 1000s of links and I often forgot some of them in my daily routine….
The SEO free resource to rank higher in the search engines. What you need to create a good site from the start.
Google Adwords Use the keyword tool to find keyword idea and better domain name. This software allows you to research keywords. It tells you the approximate common variety of searches per 30 days for a keyword phrase and in addition suggests comparable alternative keyword phrases.
Wordtracker Free Keyword Suggestion Tool. Enter some key phrases and you will receive a list of as much as one hundred associated keywords and an estimate of the number of daily searches for every keyword phrase.
sitevaluestats Get 150 backling with one click. Just enter your domain name, hit enter and let the page do his work.
247backlinks.info receive 1,000 backlinks absolutely free at sign up that you can divide in up to 25 different campaign. After that you can either buy backlinks at a really low price or refer another webmaster and earn 1000 links per referred member.
Boostability Free Website Submission to major search engines and website analysis.
socialmonkee Easily build 25 UNIQUE backlinks to your pages every day! Uses spin text for title and description. You can submit a website daily. It can be a unique site or link to an internal page. You can upgrade to pro version for a small one time fee of $47.
Backlinksseo Increase pagerank and link popularity by posting blog comments to 1000 and 1000s of popular blogs with your relevant keywords.
Pingomatic.com Ping-O-Matic is a service to update different search engines that your blog has updated. Do not abuse the system. If you run lots of blogs, do not submit each one daily, even if they are updated.
Dropmylink.com Use these different Google queries to search out thousands of web sites to drop your hyperlinks on and create backlinks.
Keyword Rank Checker Tool This website allows you to find out where your web site is ranked in Google for a specific keyword phrase. Many of the visitors goes to the sites listed on the primary page of Google, so your aim should be to get your site into the top 10. This tool means that you can check your progress extra quickly than you could do manually.
Website Grader After getting optimized your web site and started to market it, this tool is an excellent approach to get a free critique of your site. It offers information about the effectiveness of your on-web page web optimization, off-page website positioning, blogging and social marketing. It additionally gives you ideas on how you can improve in each of those areas. It allows you to measure your progress and find potential areas for improvement.
Twitter Grader Twitter Grader is a free tool that allows you to verify the power of your twitter profile compared to hundreds of thousands of other customers which were graded. Simply enter your twitter username (password not needed) and you'll get an on the spot grade and report. It is easy.
Sitemap Generation Tool This tool will generate a sitemap for you in both XML or HTML format. A sitemap contains hyperlinks to all the pages in your website. A sitemap helps to make sure that the various search engines discover all of the totally different pages on your website. It may also be useful in your visitors.
TwitterFeed feed your blog to twitter, facebook, Status.net and more. One time set up.
copyscape Search for copies of your page on the web. Especially useful if you use outsourcing resource or write original content.
worthalyzer Estimate your site value if you plan to flip it.


To ensure the reliability and resale value of your Volkswagen vehicle, trust Granby Volkswagen.
If you want to do business with a team of experts who answer your questions without putting pressure, visit us today.
We strive to exceed your expectations at Granby Volkswagen, and one of our goals is to help you save money.
Welcome to Granby Volkswagen, your dealership of choice for the purchase or maintenance of your Volkswagen vehicle in Granby. Over the years, Granby Volkswagen has become the number one destination in Granby for everything related to Volkswagen, and our team of passionate advisors and representatives is continually looking to exceed your expectations. Either by the quality of our work or our attention to customers, we know how to ensure your complete satisfaction at Granby Volkswagen.
If you're looking for a new vehicle, you'll certainly find it at Granby Volkswagen. We have an extensive inventory of Volkswagen vehicles and SUVs that have just arrived to offer you. Whatever your needs, there's a new Volkswagen vehicle that can meet your expectations, and you'll find it at Granby Volkswagen in Granby.
We also have several pre-owned Volkswagen models, many of which are certified. Our team will be happy to listen to your needs and explain the various models available to you, to discover the model that suits you best.
Whether it's a new Volkswagen vehicle or a pre-owned model, you'll find everything you need at Granby Volkswagen. We also have attractive discounts on all of our Volkswagen vehicles and services, as well as affordable financing and leasing offers on all of our vehicles for sale.


text:
He Picked His New Wife Over His Daughter — And I'm Livid
Photo: Ekaterina Pokrovsky / Shutterstock
Alex Alexander
Sep 7, 2020, 11:30 EDT
True to my mom's flair for socializing, she had dressed the house in yellow and white splendor, representing California's sunlight.
The decorations, the food — they were flawless. But hidden just below the surface was a rift of betrayal.
Just below the surface sat a dejected little girl and a horrified sibling.
RELATED: I Talked To Men Who Bailed On Their Kids — Here's What They Had To Say About Their Decision
This was my brother's going-away party, an event that — for me — was fueled with such a mixture of irritation and disbelief that I could hardly enjoy myself.
I watched everyone around me, laughing and eating, patting my brother on the back, congratulating him, listening to him wax poetic about the weather, and his new job.
But for me, in my head, I was screaming. "Don't you know what's happening?! Can't you see the problem here?!"
Because here's what was really going on: My brother, someone I respected, was abandoning his toddler daughter.
At the persistence of his new wife, he found a job in California, packed up their apartment, and moved himself, his wife, and their toddler son across the country. His daughter from his first marriage was left behind.
She has her mother, whom she loves dearly, but what about her father and half-brother? They are gone.
I will concede that in certain situations, maybe a father has no other choice but to leave. You know, like restraining orders — that seems like a fair enough reason, and one the kid benefits from.
But this is completely unnecessary. My brother, supposedly of sane mind, looked himself in the mirror and said, "Yes. I can leave my daughter. And so will her brother and stepmother. I willingly choose a job over my own flesh and blood."
RELATED: A Man Risked This Baby's Life In Order To Get 1,000 Likes On Facebook
Who does that? What kind of parent basically sells away the love of their child to the highest bidder? Someone who never should have had kids to begin with, that's who.
Call me crazy, but if you aren't willing to be there for your child, you should not have children. Without question. Do not reproduce if you do not intend to be the best parent alive.
And then, once you've successfully raised your child and head back into an empty nest, maybe then your kids can move away from you. Not the other way around.
I suppose it's just a maternal instinct to keep children safe, and maybe it doesn't cross the paternal line. But here's the fallout.
My respect for this man is gone, and I don't know how to deal with that. Someone I used to consider a personal hero has thrown his family into the mud in favor of more money.
I can no longer look him in the face, nor can I take his advice with anything other than disdain.
Many of my friends think he won't last a year without his daughter and he'll move back. But I just don't believe that. Because, as a result of this complete flop of a decision, I'm left thinking he doesn't care about her at all — and I'm willing to bet that, to an extent, she can understand the situation, she feels the same way.
The upside of all this is that, although people very dear to my niece have bailed, she still has a wonderful mother and a legion of other family members at her disposal.
We will be here to love her, support her, and see her through the strife of becoming an adult.
And you can be sure that when she asks me why daddy didn't love her enough to stick around, I'll be the first one to say, "Let's call him and ask."
RELATED: My Ex Left Me Before The Tombstone Was On Our Stillborn Son's Grave
Trending on YourTango:
10 Tips For Raising Perfect, Unspoiled, Angel Kids
10 MAJOR Parenting Fails You're Probably Guilty Of
The #1 Indicator You Were Emotionally Neglected As A Kid
7 Reasons Why I'll NEVER Spank My Kids (& Neither Should You)
Alex Alexander is a writer who focuses on relationships, health and wellness, and love.


text:
Dan Hausman
Online Papers
Philosophy of Economics
Like many of the social sciences, economics grew out of philosophy, and the concerns of economists continue to intersect with those of philosophers. Philosophical reflections on (a) scientific method and social ontology, on (b) the nature of rationality, self-interest, and preference; and on (c) welfare, justice, equality, and freedom are of abiding significance to economists and other social scientists.
1. Economic methodology
Philosophical reflection on economics is ancient, but the conception of 'the economy' as a distinct object of study dates back only to the 18th century. Aristotle addresses some problems of economics mainly as problems of household management. Scholastic philosophers addressed ethical questions concerning economic behavior, and they condemned 'usury' -- that is, the taking of interest on money. With the increasing importance of trade and of nation-states in the early modern period, 'mercantilist' philosophers and pamphleteers addressed questions concerning the balance of trade and the regulation of the currency. Only in the work of the physiocrats and especially of Adam Smith do scholars begin to think of the economy as an object of study with its own principles and laws.
18th-century philosophers wrote in the shadow of Newton's towering accomplishments. David Hume is unabashed about his hopes to develop a science of mind and society in the image of Newton's science of the solar system. To that end, he seeks out general laws of individual thought and action, out of which larger-scale orderly relations will arise, in just the way that patterns in planetary motion arise from the laws of motion and gravitation governing individual bodies. Thus Hume traces the rise in prices and the temporary increase in economic activity that follow an increase in currency to the perceptions and actions of individuals who first spend the additional currency.
It then lay to Adam Smith to extend such a method to a systematic Inquiry into the Nature and Causes of the Wealth of Nations and to make explicit an implication of previous causal inquiries such as those of Hume, the physiocrats and of many of the mercantilists. That implication (which is arguably the foundation for all social science) is that the social, aggregative implications of individual choices are often unintended. In spending their additional gold imported from abroad, traders do not intend to increase the price level. But that is what they do nevertheless. In this way unintended regularities that arise from the intentional choices of individuals can rule those individuals just as surely as the regularities of nature, and, like nature, societies can be objects of scientific investigation.
1.1 Classical economics and the method a priori
Although Adam Smith made some general comments about scientific method in his History of Astronomy and other essays, he wrote little explicitly about the method of economics. The first extended reflections on economic methodology had to wait until Nassau Senior (1836) and John Stuart Mill (1836). Their essays must be understood against the background of the prevailing economic theory. Like Smith's economics (to which it owed a great deal) and modern economics, the 'classical' economics of the middle decades of the 19th century traced economic regularities to the choices of individuals facing social and natural constraints. But, as compared to Smith, more reliance was placed on severely simplified models. In David Ricardo's Principles of Political Economy, a portrait is drawn in which wages above the subsistence level lead to an expanding labor force, which in turn requires more intensive agriculture or cultivation of inferior land. The extension of cultivation leads to lower profits and higher rents; and the whole tale of economic development leads to a gloomy stationary state in which profits are too low to command any net investment, wages return to subsistence levels, and only the landlords are affluent. From the time Ricardo's Principles was published (1819) the data available to classical economists was never in accordance with the trends the theory predicted. Yet the theory continued to hold sway for more than half a century, and the unfavorable data were explained away as due to various 'disturbing causes.'
It is not surprising then that Mill's account of the method of economics would emphasize the relative autonomy of theory. Mill distinguishes between two main kinds of inductive methods. The method a posteriori is a method of direct experience. It is only suitable to phenomena in which few causal factors are operating or in which experimental controls are possible. Mill's famous methods of induction are detailed specifications of the method a posteriori. In his method of difference, for example, one holds fixed every causal factor except one and checks to see whether the effect ceases to obtain when that one factor is removed.
Unfortunately, such direct inductive methods cannot be used to study phenomena in which many causal factors are in play. If, for example, one attempts to investigate whether tariffs enhance or impede prosperity by comparing the prosperity with and without tariffs, the results will be irregular and unreliable because other causes besides tariffs will also differ across societies and times. So one needs instead to employ the method a priori. Despite its name, this is an inductive method, but it is an indirect inductive method. One first determines the laws governing individual causal factors in domains in which direct inductive methods are applicable. Having then determined the laws of the individual causes, one investigates their combined consequences deductively. Finally, there is a role for 'verification' of the combined consequences, but owing to the causal complications, this testing has comparatively little weight. The testing of the conclusions serves only as a check on one's deductions and as a indicator of whether there are significant disturbing causes that one has not yet accounted for. Mill gives the example of the science of the tides. One determines the law of gravitation by studying planetary motion, in which gravity is the only significant causal factor. Then one develops the theory of tides deductively from that law and information concerning the positions and motions of the moon and sun. The implications of the theory will be inexact and sometimes badly mistaken, because many subsidiary causal factors influence tides. By testing the theory one can uncover mistakes in one's deductions and evidence concerning the role of the subsidiary factors. Because of the causal complexity such testing does little or nothing to confirm or disconfirm the law of gravitation, which has already been established.
Because economic theory includes only the most important causes and necessarily ignores many minor causes, its claims, like claims concerning tides, are inexact. Its predictions will be imprecise, and sometimes dead wrong. But it is possible nevertheless to develop and confirm economic theory by first establishing in simpler domains the laws governing the major causal factors and then deducing their consequences in different circumstances. For example, statistical data tell a mixed story about the relationship between minimum wages and unemployment; and there is no data at all about what the consequences for employment would be of an extremely high minimum wage. On the other hand, everyday experience teaches one that firms can choose among more or less labor-intensive processes and that a high minimum wage will make more labor-intensive processes more expensive. Since one also has good reason to believe that firms try to keep their costs down, one has good reason to believe that a high minimum wage will increase unemployment.
According to Mill, economics is not only inexact and committed to the method a priori. In addition, he maintains that it is a separate science. What distinguishes economics as a discipline is not only its concern for a certain domain of phenomena, but also its limitation to a particular set of causal factors, which predominate in this domain. With respect to this domain, one can (at a certain level of approximation) ignore the myriad causal factors that influence all social phenomena and which are the subject matter for sociology and develop economics separately. In defending a view of economics as in this way inexact and separate and a view of economists as following the method a priori, Mill was able to reconcile his empiricism and his commitment to Ricardo's economics.
Although Mill's views on economic methodology were challenged later in the 19th century by dissident economists who believed that the theory was too remote from the contingencies of policy and history, Mill's methodological views dominated the mainstream of economic theory for well over a century. Mill's vision survived the transformation of economics from classical to neoclassical and is clearly discernable in the most important methodological treatises concerning neoclassical economics, such as John Neville Keynes' The Scope and Method of Political Economy (1891) or Lionel Robbins' An Essay on the Nature and Significance of Economic Science (1935). Indeed Hausman (1992) argues that current methodological practice closely resembles Mill's methodology.
1.2 20th-century philosophy of science and economic method
Beginning in the 1930s, mainstream economists began to have a bad conscience about their traditional methodology, which some saw as insufficiently empiricist. Robbins is a transitional figure, because at the same time that he clung to Mill's methodology (1935, chap. 4) and to the identification of the domain of economics with the predominance of particular causal factors (the allocation of scarce means that have alternative uses -- 1935, chap. 1), he also offered his apparently empiricist critique of interpersonal utility comparisons as untestable value judgments (1935, chap. 6). Terence Hutchison argued that the propositions of pure theory were so hedged with ceteris paribus conditions that they were not testable (1938). Paul Samuelson argued for a need to separate the 'operationally significant' economic wheat from the chaff, and in his theory of revealed preference he provided a model for how to do so (1947). Other economists cited survey data to argue that the theoretical propositions of economics were false. The confusing methodological situation stabilized in the 1950s with arguments by Fritz Machlup (1955) and especially Milton Friedman (1953) that economists need be concerned to fit only data concerning prices and quantities and that the 'realism of assumptions' is irrelevant.
Although confused, mistaken, and inconsistent with the practice of mainstream economists--including his own practice--Friedman's views have dominated the methodological conceptions of mainstream economists over the past two generations. His views are confused because they conflate many different kinds of 'assumptions' and 'realism'. Basic generalizations, propositions concerning initial conditions, and antecedents of conditional claims are all called 'assumptions'. Assumptions are called 'unrealistic' if they are false, incomplete, or not approximately true. Friedman's views are mistaken, because (as Mill already emphasized) market data are generated by too many causal factors to provide effective tests of economic theory. And the eclecticism that Friedman's official methodological position counsels--'don't worry about what theory says, just ask whether it fits market data'--contradicts the firm and narrow commitment to a specific theory that characterizes mainstream economics, including Friedman's own work. Nevertheless Friedman's views were warmly embraced because they freed mainstream economists to ignore criticisms of their theories and to abandon all empirical work apart from econometrics.
In the 1970s and 1980s other streams of contemporary philosophy of science began to influence the methodological reflections of economists. Karl Popper's view that scientific theories must be falsifiable, which had been defended by Hutchison was taken up and emphasized by Mark Blaug (1992). According to Popper, the hallmark of science is to formulate theories so that they can be exposed to empirical testing and to reject theories that fail tests. Hutchison and Blaug object that mainstream economic theories are not harshly tested and that mainstream economists are unwilling to surrender them when they fail the few tests they are subjected to. In his extensive publications, Lawrence Boland (1982) has also emphasized the relevance of Popper's views to economic methodology, although Boland's interpretation of Popper is rather different than Blaug's or Hutchison's.
Imre Lakatos' methodology of scientific research programmes (which represented a wedding of some of Thomas Kuhn's insights to Popper's philosophy of science) was for a while widely discussed (Latsis 1976, deMarchi and Blaug 1993). Although still emphasizing the importance of empirical criticism, Lakatos insisted that theories should not be abandoned until superior alternatives are found, and his emphasis on heuristics struck a responsive chord.
A spate of prominent contemporary writers on methodology: Roger Backhouse, Bruce Caldwell, Neil deMarchi, D. Wade Hands, and E. Roy Weintraub have a more ambivalent relationship to Popper's and Lakatos' philosophy. Caldwell wrote a series of searching, though charitable criticisms of Popperian views. Backhouse, DeMarchi, Hands, and Weintraub were all at one time enthusiastic about Lakatos' views, though to differing degrees all have moved away from them now. Backhouse and deMarchi remain the closest. Hands has become a proponent of the application to economics of new work on the sociology of science, while Weintraub's work on the stabilization of concepts in economics has been more influenced by literary theorists
More recently, alternatives to philosophy of science influenced by work in literary theory and sociology have also developed a following among economists. In a series of stylistically brilliant works, Deirdre McCloskey has criticized the whole project of exploring a normative methodology for economics and has urged instead that economists attend to their rhetoric--that is, to their ways of persuading one another (1985). This work has been highly controversial, because many of McCloskey's formulations apparently reduce questions about the correctness or incorrectness of economic claims to questions about what most economists accept. Such a view would imply that minority views are always mistaken. McCloskey denies such a radical reading of her position, but it is not clear how to avoid it without incorporating normative methodological commitments into the rhetoric of economics.
Philip Mirowski's work on methodology is heavily historical and less radical epistemologically than McCloskey's. He has explored the influence on economics of the formal analogy between utility theory and physics (1990). More recently, his work has shown increasing concern with sociological influences. For other work that tests the boundaries of conventional methodology, see Mäki, Gustafsson and Knudsen 1993.
Over the last fifteen years, economic methodology has become a large field. It has its own Journal of Economic Methodology and occupies a large part of the journal, Economics and Philosophy. Dozens of monographs on economic methodology have been published. Graduate programs in the field have been established (including a Ph.D. program at Erasmus University in Rotterdam), and there are regular sessions on economic methodology at meetings of economists and philosophers. The field is very diverse, and each of its leading figures has a distinctive approach. Although one can roughly identify Popperian-Lakatosian and sociological-literary schools, there is no easy way to categorize contemporary methodology.
Several methodologists have been particularly interested in the role of explicitly causal notions in economics (see Causes and laws; Causation: physical, mental and social). Nancy Cartwright (1989), who is also a distinguished philosopher of physics, has (unlike most philosophers writing on economics) paid a great deal of attention to econometrics. She has defended the importance of specifically causal considerations in science, and she has argued that econometricians have made important contributions to a philosophical understanding of causation. Cartwright holds that economists should be understood as attempting to identify causal capacities and that the work of econometricians can be understood as contributing to this task.
Daniel Hausman (1992) defends a modernized variant of Mill's methodology that depicts economics as inexact and permits the plausibility of the basic principles of economics to count in the assessment of economic theories. But he is skeptical of the view that a small set of causal factors do indeed predominate in the domain of economics and defends a greater attention to the results of experiments, surveys, and field work. He has increasingly emphasized the role of causal generalizations in mainstream microeconomic theory.
Kevin Hoover (2000)-- unlike most methodologists -- has written on problems concerning macroeconomics. Although he has addressed technical issues in econometrics, his main interest has been in causality. Like Cartwright, Hoover has placed particular emphasis of what one can learn about causation by studying the work of economists.
Tony Lawson (1997) finds his inspiration in the 'transcendental realism' of Roy Bhaskar. Lawson sees himself not merely as enunciating a distinctive approach to methodology, but as laying the foundations for a new economics. Crucial to the approach and to his methodological views is a strongly realist ontology (see Realism, instrumentalism, fictionalism), which takes the objects of scientific investigation to be causal mechanisms and tendencies that lie as it were 'beneath' the irregularity of phenomena. This realist ontology leads him, like Cartwright, Mäki and increasingly Hausman, to regard economic theories as identifying causal mechanisms and tendencies.
Uskali Mäki is harder to categorize. Like Lawson, he is concerned with realism, but whereas Lawson attempts to redirect economics and its methodology on the basis of a specific realist ontology, Mäki elucidates versions of realism to which economists have been implicitly committed. Like Cartwright, Hausman and Hoover, Mäki emphasizes the importance of causal notions in economics. Like Hands and Mirowski, Mäki also applies insights from the sociology of science to the understanding of economics.
There is a great deal of other work, too. A number of economists and philosophers have attempted to apply to economics the structuralist view of scientific theories developed by Patrick Suppes, Joseph Sneed, and Wolfgang Stegmüller. (See for example Hands 1985 and Balzer and Hamminga 1989.) Alexander Rosenberg, who is also a distinguished philosopher of biology, has pursued a number of different themes. His Microeconomic Laws: A Philosophical Analysis (1976) was one of the first philosophical treatments of the methodological problems of economics. In that book Rosenberg argues that economics fits reasonably well within a standard philosophical model of the natural sciences. But shortly afterwards Rosenberg changed his views radically, and he now defends the position that mainstream economics is best understood either as applied mathematics or as a normative social and political theory (1992).
Contemporary economic methodology thus moves in many directions. Although some of it is directed toward the conduct of economics, one regularly finds a sort of methodological schizophrenia whereby in theory economists cling to positivist or Popperian philosophical views that are radically at odds with their practice, which is roughly Millian. Other work in methodology is oriented toward philosophy of science, and it appears that the study of economic methodology has made significant contributions to philosophical investigations of causation and explanation. It is controversial whether methodology can or should be a distinct and relatively self-contained field rather than addressing itself toward economists or toward philosophers.
2. Methodology, rationality, preference, and self-interest
The study of rationality, preference, and self-interest is a second area of overlap between economics and philosophy. Mainstream economic theory is built around a variant of 'folk psychology'. According to folk psychology, human actions are the consequence of beliefs, desires, and circumstances that determine the feasibility and consequences of actions. The same elements figure in economic theories of choice. Sometimes economists suppose that agents have complete knowledge and thereby avoid having to refer specifically to beliefs, but in other contexts beliefs are explicitly modeled--most typically as subjective probability judgments. In place of desires, economists postulate that economic agents possess stable, complete, and transitive preferences. Given additional technical conditions, such preferences can be represented by a continuous 'utility function' such that UP(A) > UP(B) if and only if an agent P prefers A to B. 'Utility' is simply a way of indicating how an alternative is ranked by an agent. It is not a substantive object of choice, and indeed it makes no sense to regard an agent as seeking or preferring utility. Mainstream economic models relate choice to preference by maintaining that, subject to the constraints, agents maximize utility -- which means merely that among the feasible alternatives agents choose whatever they most prefer.
The previous paragraph presents the standard theory of choice as an empirical theory purporting to describe, predict, and explain how agents choose in fact. But the same theory also functions as a theory of rationality: Preferences are rational if they are complete and transitive (and perhaps satisfy other axioms, too). Degrees of belief are rational if they satisfy the axioms of the calculus of probability. Choices are rational if they are utility maximizing. Given this model of rationality, the empirical claim of the standard theory of rational choice can be restated simply as the claim that economic agents are in fact rational.
According to this theory of rationality, it is irrational to choose one feasible option over another when one prefers the second and according to this theory of choice people will in fact never choose one feasible option over another if they prefer the second. Since according to these theories choice is and ought (rationally) to be determined by the agent's preferences, it is easy mistakenly to infer that these theories maintain that agents are and ought to be self-interested. But whether a choice is self-interested depends on the content of the preferences that lead to it, not on whether the choice is determined by preferences. People who prefer to sacrifice their interests to others are not self-interested, and that fact that their choices result from these preferences does not make their choices self-interested. Unlike the standard theories of choice and rationality, the views that it is rational to pursue one's own interest and that people in fact do so are 'substantive' theories of choice and rationality. Unlike the standard 'formal' theory, which merely specifies a certain structure of choice and preference, self-interest theories specify an objective that agents do or ought rationally to have.
Theories of rationality are normative theories. They prescribe what one ought to do--not, to be sure, as a matter of morality, but as a matter of rationality or prudence. (Irrationality is foolish rather than evil.) Incorporating as it does a normative theory, economics is unlike any of the natural sciences. The reason why economics incorporates a theory of rationality is that human actions, unlike the actions of oak trees or potassium, can be criticized or justified as well as explained. This fact has important methodological consequences and indeed (as just argued) establishes one important distinction between the social and the natural sciences.
The first and perhaps most important methodological consequence is that explanations of individual choices cite the reasons why the agent acted. Although a number of philosophers in the 1950s, who were influenced by Wittgenstein, argued that such explanations that cite an agent's reasons could not be causal explanations, nowadays most philosophers are persuaded by Donald Davidson's argument (1963) that satisfactory explanations that cite an agents reasons must also be causal explanations. Agents may reasons for performing an agent that are not in fact responsible for the action. Davidson argued that the distinction between the reasons that are 'effective' and those that are merely rationalizations is that the first unlike the second are causes of the action.
So the fact that the explanations economists offer of individual choices cite reasons for those choices does not imply that economists are not giving causal explanations, and it does not imply a strong anti-naturalist distinction between the structure of the natural and social sciences. But the fact that the causal explanations economists offer also cite reasons is nevertheless of great interest. It explains why economists must hold that agents are to some extent rational. (If they weren't then they couldn't have reasons for their actions.) Furthermore, it implies that the actions of economic agents are subject to rational appraisal. Were the agent's reasons good reasons? Was the action justified? And once one begins appraising choices, one is just one step away from ethical questions. Readers should be aware that the standard theories of rationality and choice are controversial. For more detail on the theories and these controversies, see Intentionality and rationality; and Rational choice explanations.)
3. Welfare, justice, equality, and freedom
Economists have an ambivalent attitude toward ethics. On the one hand, many have been concerned to insist that mainstream economics is a 'positive' science, whose conclusions are entirely independent of any moral commitments. On the other hand, economists freely give out normative policy advice. Some of this advice is purely technical, like the advice that a civil engineer might offer on where to locate a bridge, but much of it is not. Most mainstream economists are in fact committed to a specific view of ethics that emphasizes welfare, and they also adopt a distinctive theory of welfare. (When economists emphasize efficiency they are virtually always concerned with efficiency at promoting individual welfare.) These features of normative economics are not entailed by positive economics or the theory of rationality, but they are heavily influenced by them.
There are a variety of different ethical bases for assessing social arrangements. In addition to considering the welfare of the members, one can also ask whether the rights of the members are protected and whether procedures and the distribution are just, whether the members are treated equally, and what sorts of freedoms and opportunities people enjoy (see Rights; Justice; Equality; Liberty/freedom). All of these considerations are important, but in economic policy pronouncements and normative theory (which indeed is called 'welfare economics') typically only considerations of welfare enter.
The reason for this narrow normative focus is that welfare can be tied tightly to the standard models of rationality and choice. Suppose that individuals are rational and--in addition--that they are self-interested. As already pointed out, the standard theory of rationality does not imply self-interest, but self-interest is also commonly assumed in most economic models. It follows that individuals will prefer alternative A to B if and only if they believe that A serves their interests better than B. If individuals have perfect knowledge, which is again a common assumption in economic models, it follows that individuals will prefer A to B if and only if A in fact serves their interests better. Finally, if one identifies an individual's welfare with whatever is in the individual's interest, it follows that A is better for and individual than B if and only if the individual prefers A to B--or in other words that welfare is the satisfaction of preference. This view of welfare has the additional advantage that it prevents concerns about paternalism--to which many economists are opposed--from even arising, since by definition it could never be good for individuals to overrule their preferences.
Most normative economics is thus doubly narrow. Not only is it characterized by an almost exclusive preoccupation with welfare, but it is also committed to a particular view of welfare as the satisfaction of preferences. This double narrowness is doubly unfortunate. The limitation to considerations of welfare means that in policy discussions economists treated non-welfarist ethical criteria as exogenous constraints to be left for someone else to worry about. The commitment to a preference satisfaction view of welfare has the drawback that this view of welfare is false. Because individuals are not always self-interested and because their beliefs are not always true, individuals do not always prefer what is good for them. There are a variety of good reasons to be hesitant about paternalism, but the view that individuals are always perfect judges of their own good is not one of them. There is some work, particularly by Amartya Sen (see for example his 1992), that stretches these limits on normative economics, but such work is the exception to the welfarist rule (see also Hausman and McPherson 1996; Economics and ethics and Welfare).
Mainstream economists, whether engaged with positive theory, the theory of rationality, or with normative and policy investigations, are overwhelmingly committed to an exaggerated simplification of a plausible view of individual agents as possessing stable and consistent preference rankings, which, given the constraints, determine their choices. Market outcomes are the unintended consequences of these choices. Rationality is defined by this structure of choice. Welfare, like choice, is determined by these same preference rankings. Although there are philosophical issues concerning economics that have no particular connection to this core commitment, most work in philosophy of economics has attempted to comprehend and evaluate the basic model of choice and its applications in positive economics, normative economics, and the theory of rationality.
Backhouse R 1997 Truth and Progress in Economic Knowledge. Edward Elgar, Cheltenham.
Balzer W, Hamminga B eds. 1989 Philosophy of Economics. Kluwer-Nijhoff, Dordrecht.
Blaug M 1992 The Methodology of Economics, 2nd. ed. Cambridge University Press, Cambridge.
Boland L 1982 The Foundations of Economic Method. Allen & Unwin, London.
Caldwell B 1982 Beyond Positivism: Economic Methodology in the Twentieth Century. Allen & Unwin, London.
Cartwright N 1989 Nature's Capacities and their Measurement. Clarendon Press, Oxford.
Davidson D 1963 Actions, reasons and causes. J. Phil. 60: 685-700.
de Marchi N, Blaug M eds. 1991 Appraising Economic Theories: Studies in the Methodology of Research Programs. Edward Elgar, Cheltenham.
Friedman M 1953 The methodology of positive economics. In: Essays in Positive Economics. University of Chicago Press, Chicago, pp. 3-43.
Hands D W 1985 The structuralist view of economic theories: the case of general equilibrium in particular. Econ. Phil. 1: 303-35.
Hands D W 1993 Testing, Rationality, and Progress: Essays on the Popperian Tradition in Economic Methodology. Rowman & Littlefield, Lanham, MD.
Hausman D 1992 The Inexact and Separate Science of Economics. Cambridge University Press, Cambridge.
Hausman D, McPherson M 1996 Economic Analysis and Moral Philosophy. Cambridge University Press, Cambridge.
Hoover K 2000 Causality in Macroeconomics. Cambridge University Press, Cambridge.
Hutchison T 1938 The Significance and Basic Postulates of Economic Theory. Repr. A. M. Kelley, New York, 1960.
Latsis S ed. 1976 Method and Appraisal in Economics. Cambridge University Press, Cambridge.
Lawson T 1997 Economics and Reality. Routledge, London.
McCloskey D 1985 The Rhetoric of Economics. University of Wisconsin Press. Madison.
Machlup F 1955 The problem of verification in economics. Southern Economic Journal 22: 1-21.
Mäki U, Gustafsson B, Knudsen C eds. 1993 Rationality, Institutions and Economic Methodology. Routledge, London.
Mill, J. S. 1836. On the definition of political economy and the method of investigation proper to it. Repr. Collected Works of John Stuart Mill, vol. 4. University of Toronto Press, Toronto, 1967.
Mirowski P 1990 More Heat Than Light. Cambridge University Press, Cambridge.
Robbins L 1935 An Essay on the Nature and Significance of Economic Science, 2nd. ed. Macmillan, London.
Rosenberg A 1992 Economics--Mathematical Politics or Science of Diminishing
Returns, University of Chicago Press, Chicago.
Samuelson P 1947 Foundations of Economic Analysis. Harvard University Press., Cambridge, MA:
Sen 1992 Inequality Reexamined. Harvard University Press, Cambridge, MA.
Senior N 1836 Outline of the Science of Political Economy. Repr. A. M. Kelley, New York, 1965.
Weintraub E R Stabilizing Dynamics. Cambridge University Press, Cambridge.
Email: dhausman@wisc.edu
5197 Helen C. White Hall
600 N. Park Street, Madison, WI 53706-1474


How can I create a recursive routive that goes through ALL objects in a document?
I want to go through ALL... even all children.
But now, my problem is another... it has to do with opening the hierarchy of any object... I can't ask too much of myself... I'm no coder ;-) But this seemed like a simple thing to do.
This is not the same I gave you at first. It works vice versa. Set the Bit to open the hierarchy and delete the bit to close/fold it. I just tried it, it works perfectly.
I will try it, Samir.
I would like to know how to search the entire document as well.


text:
Newsom's Climate Order Snubs Environmental Justice Advocates and Protects Big Oil
Governor Newsom knows how to fool some of the people with his most recent symbolic announcement, but we see through the fluff. Here's the real scoop on his Executive order.
Post date Sep 25, 2020
This week, California Governor Gavin Newsom convened a press conference, stood in front of the cameras and signed an executive order proclaiming California's bold climate leadership. It earned him praise and positive headlines across the world. But like Newsom's previous climate declarations, his announcement was big on symbolism and short on substance. He codified things already in progress and directed agencies to do things they were already doing. Rather than leading, Newsom kicked major decisions to the legislature. Instead of taking bold actions demanded by science, Newsom gaslit activists. He called us "emotional" and, ironically, disparaged our demands as "symbolic."
The bottom line: Newsom's much-hyped proclamation on electric vehicles is mediocre and his actions on oil and gas drilling are an insult to those fighting for clean air, clean water and their lives during this pandemic.
The best part of Newsom's "new" order were already largely underway
First, let's talk about the shiny object — the part of his Executive Order that was decent. Newsom announced a ban on the sale of new cars with internal combustion engines and mandated that all light duty vehicles sold by 2035, and all heavy duty vehicles sold by 2045 must be 100% electric. This is important because transportation accounts for 41% of California's greenhouse gas emissions. It's also important because cars consume refined oil; reducing this demand is key to ending fossil fuel extraction.
But this fails the test of bold leadership for two reasons. First, California was already on track to do this. The California Air Resources Board was already working toward adopting these goals, and in developing new rules, their own technical experts said 2035 is not fast enough. Second, an all electric fleet will only have a significant climate impact if the fuel that powers the electricity is 100% renewable. Unfortunately, California's 2045 goal for achieving 100% renewable energy is far too late a deadline, given the enormity of the climate crisis. It's disappointing to see that Governor Newsom wasted an opportunity to accelerate these targets. He pursued symbolic action over substantive policy.
Issue by issue, Newsom has failed to accomplish what we've asked of him
Meanwhile, Governor Newsom's actions on fossil fuel supply — mainly oil and gas drilling — were a complete mess. Newsom's actions ignored climate science and ignored the urgency of the health crisis faced by families living near oil wells. Newsom failed to deliver a single thing Food & Water Watch/Action and our allies with Last Chance Alliance asked of him.
Here's the breakdown of what we asked Newsom to do versus what he actually did.
Stop issuing new oil and gas permits. Newsom has issued over 1,500 permits for new wells, and 2,673 total in 2020. His total since being in office is over 7,000 permits! California has over 80,000 active oil and gas wells, and adding over 1,500 more is unacceptable. It is extremely problematic to allow the fossil fuel industry to expand drilling during a climate emergency and global pandemic. We need to move off fossil fuels now. Approving new wells just makes things worse.
Newsom's Executive order did nothing to reverse course on his doubling down on expanding fossil fuel production. Worse, during his press conference, Newsom stated that if California stopped drilling for oil, we would have to import more oil. He left out a very important fact that California's oil imports are largely used to refine oil for gasoline and other fuels that we then export. Governor Newsom continues to approve increases in drilling and refinery production while finding any excuse not to curb either.
Ban fracking. Newsom promised to ban fracking while running for Governor. Data from 2001-2015, shows about 20% of California oil comes from fracked wells according to a state study. Banning fracking would have significant impacts on protecting our climate, water and air as fracking spews methane and toxins into the air and leaks can poison water.
Newsom's Executive order stated that he wants the legislature to ban fracking by 2024. Then during his press conference he doubled down and falsely asserted he does not have the authority to ban fracking — passing the buck to the legislature. Newsom's authority to act, however, is clear. Director of the Stanford Law School Environmental Law Clinic, Deborah Sivas, wrote in a recent OpEd that the oil industry does not have the right to more drilling permits or the right to drill using risky practices like fracking. Further, Governor Newsom is the boss of the California Geologic Energy Management Division (CalGEM), CalGEM issues drilling and fracking permits, and Newsom can tell them to stop. It's that simple. In fact, it's just what Newsom did last year when he temporarily halted approval of fracking permits. Newsom is not only contradicting legal experts — he's contradicting his own prior actions!
It's worrisome that Newsom directed the legislature to take on a fracking ban bill when he has the authority to do it himself, but even more troubling since he knows the oil and gas industry has enormous power to kill this bill. Year after year, the Oil Lobby with the Western States Petroleum Association at the helm is the biggest lobby in California. Oil money has easily defeated fracking moratorium bills in the past and just defeated a setbacks bill in August 2020. Only a handful of Oily Dems are needed to kill legislation, and the Oil Lobby has them lined up around the block.
Newsom also astonishingly downplayed the impact of fracking. In speech after speech he claimed fracking represents only 2% of California drilling. The truth: on average, about 20% of California's oil came from fracking from 2001 to 2015, according to a state report. It seems unlikely that it's now just 2% only five years later — but if it is, why not just ban fracking now? A ban on fracking would be bold substantive action — a sharp contrast to Newsom's approach.
Institute a 2500 foot health and safety buffer for oil and gas wells between sensitive sites like homes and schools (existing and future).
Newsom's Executive order directed CalGEM to propose a draft rule on health and safety regulations by Dec 31, 2020. However, this order omitted a keyword, "setbacks," – the key demand from the environmental justice community and a necessary mechanism to protect public health. Environmental justice communities and advocates have campaigned for a 2500ft setback between oil operations and sensitive sites like homes and schools. Gov. Newsom has never supported this demand despite his initial promise when running for Governor to "oppose dangerous drilling operations". Newsflash: any drilling near homes, schools and hospitals is dangerous. Newsom again ignored this common sense request and left the door open for the oil industry to weaken this rulemaking process so severely rendering it pointless. Finally, CalGEM is already in the middle of a rule making process it initiated almost one year ago in November 2019. Newsom's order essentially directs CalGEM to continue what they were already doing.
Phase out oil drilling in California with a just transition for workers and communities.
Newsom's Executive order directed the CalEPA and CA Natural Resources Agency to "expedite regulatory processes to repurpose and transition upstream and downstream oil production facilities, while supporting community participation, labor standards,and protection of public health, safety and the environment." He required regulatory changes to this effect by July 15, 2021. This action is a far cry from a mandate to transition California off oil. Any number of policies could result from this, making it purely symbolic and another sidestep of responsibility to define bold goals. As Newsom sets timelines for reducing our reliance on oil via electric vehicle mandates with clear deadlines, Newsom must address the other side of the equation. He must manage the decline of drilling and set a deadline for this transition.
The bottom line is Governor Newsom is playing a political game with the lives and sustainability of California's people and resources. We aren't fooled, we demand better, and we hope that you will too.
More people need to know about this.
Jan 28, 2022 Court Halts Gulf Oil and Gas Leases Where Biden Administration Failed to Lead
Jan 27, 2022 One Year Later, Biden Climate Record is Rhetoric Over Action
Jan 26, 2022 California Rejects Petition to Drop Factory Farm Gas From Energy Credit System
← This Is How We Pry The Corporate Stranglehold Off Of Our Farm System → Gov. Newsom Fails on Real Climate Action, Once Again