Engineering a Training Data Set from Garbage

Hash.ai Blog: https://hash.ai/blog

Debugging

Debugging models can be difficult at times. agreed, we’re going to be adding some features and better error messages. One tip: you can use console.log or print statements and they’ll output to the dev console

Get Timestep

https://hackernoon.com/how-to-build-a-sir-model-in-20-minutes-se1m3ypg

function behavior(state, context) {
  let timestep = state.get("timestep");

  if (state.get("status") == "infected" && timestep > state.get("recovery_timestep")) {
    state.set("status", "recovered");
    state.set("color", "grey");
  }

  timestep += 1
  state.set("timestep", timestep)

}

Run Once

Probably the most straightforward way to make sure a behavior only runs once is by modifying the agents behavior field after the ‘business logic’ runs. So for example if your agent has behaviors = [“foo”, “bar”] you can set the behaviors as state.set(“behaviors”, [“foo”])

Additionally one way to prevent things running every turn is to add a timestep field to the agent , and then add a line in the code liketimestep % 2 != 0 return state And to send the agents neighbors to the creator agent you’d want to follow a messaging pattern (pseudocode)

messages.push({  
to: "creator_agent_name",
type: "followers",
data: { "followers": [ ]}
})

Code

init.json

[
  { 
    "behaviors": [
      "create_cpus.js",
      "create_users.js",
      "create_load_balancer.js",
      "create_connections.js"
    ]
  }
]

globals.json

{
   "cpu": {
      "nodes": 4,
      "cpus_per_node": 15
   },
   "user": {
      "concurrent": 3   
   },
   "request": {
      "MAX_QUERIES_PER_REQUEST": 20,
      "MAX_PROCESSING_SECONDS": 500
   }
}

create_users.js

/**
/**
 * @param {AgentState} state
 * @param {AgentContext} context
 */
const behavior = (state, context) => {
  
  // Creation should only run once.
  if(context.currentStep != 1) { return state; }
  
  const globals = context.globals();
  let messages = state.get("messages");

  const PLACEMENT_FROM_CENTER_Y = -12;
  const PLACEMENT_FROM_CENTER_X = globals["user"]["concurrent"] % 2 == 0 ? 0 : 1
  let agents = state.get("agents") || [];
  
  for(user_i = 0; user_i < globals["user"]["concurrent"]; user_i++){

    // Calculate starting position.
    let y = user_i % 2 == 0 ? (user_i * -1): user_i;
    y = y % 2 != 0 ? y + 1 : y;
    const x = PLACEMENT_FROM_CENTER_Y;
    const z = 0;

    // Define agent.
    const cpu_agent = {
        "agent_name": `user_${user_i}`,
        "behaviors": ["user.js"],
        "position": [x, y + PLACEMENT_FROM_CENTER_X, z],
        "color": "yellow",
        "waiting": false
    }
    agents.push(cpu_agent);

    // Prepare Hash create agent commands.
    messages.push(
      {
        "to": "hash",
        "type": "create_agent",
        "data": cpu_agent
      }
    )
  }

  // Request Hash create agent.
  state.set("messages", messages)
  // Store agents.
  state.set("agents", agents)

  console.log("Created users.");
  
  return state;
};

create_load_balancer.js

/**
 * @param {AgentState} state
 * @param {AgentContext} context
 */
const behavior = (state, context) => {
    
  // Creation should only run once.
  if(context.currentStep != 1) { return state; }

  const messages = state.get("messages");

  const PLACEMENT_FROM_CENTER = 0;
  let agents = state.get("agents") || [];


  // Calculate starting position.
  const y = 0;
  const x = PLACEMENT_FROM_CENTER;
  const z = 0;

  // Define agent.
  const lb_agent = {
      "agent_name": `load_balancer`,
      "behaviors": ["lb.js"],
      "position": [x, y, z],
      "in_use": false,
      "color": "purple",
      "assignment_index": 0
  }
  agents.push(lb_agent);

  // Prepare Hash create agent commands.
  messages.push(
    {
      "to": "hash",
      "type": "create_agent",
      "data": lb_agent
    }
  )

  // Request Hash create CPUs.
  state.set("messages", messages)
  // Store agents.
  state.set("agents", agents)
    
  
  console.log("Created load-balancer.")
  return state;
};

create_connections.js

/**
 * @param {AgentState} state
 * @param {AgentContext} context
 */
const behavior = (state, context) => {
  // console.log("CONNECTION #######")

  let msgs = [];
  msgs = context.messages().filter(m => m.to[0].includes("_to_"));
  let scale = state.get("scale");

  if (msgs.length > 0) {
    // Forward CPU requests.
    let messages = forwardCPURequests(state, msgs, context);
    state.set("messages", messages);

    state.set("color", "pink");
    scale[0] = 0.5;
    scale[2] = 0.5;
  } else {
    state.set("color", "blue");
    scale[0] = 0.05;
    scale[2] = 0.05;
  }

  state.set("scale", scale);
  // console.log("END CONNECTION #######")
  return state;
};


function forwardCPURequests(state, msgs, context) {
 
  const msgsToLB = msgs.filter(m => m.to[0].includes("to_lb"));
  const msgsToNode = msgs.filter(m => m.to[0].includes("to_node"));
  
  let messages = [];

  // Forward CPU requests to load-balancer.
  for(i = 0; i < msgsToLB.length; i++){
    const m = msgsToLB[i];
    messages.push({
      "to": "load_balancer",
      "type": "cpu_request",
      "data": m["data"]
    });
  }

console.log(msgsToNode.length); 
  // Forward CPU requests to nodes.
  for(i = 0; i < msgsToNode.length; i++) {
    const m = msgsToNode[i];
    console.log(m);
    distributeReqOnNode(state, context, m, messages, context.globals());
    console.log("here");
  }

  // return messages;
  return messages;
}

function distributeReqOnNode(state, context, cpuRequest, messages, globals) {
  const agentName = state.get("agent_name");
  const nodeNum = parseInt(agentName.split("lb_to_")[1].replace("node_", ""));
  
  // const cpusRequested = cpuRequest["data"]["load"];
  const user = cpuRequest["data"]["user"];
  const cpusRequested = cpuRequest["data"]["request"];

  const cpusInNode = context.neighbors()
                          .filter(n => n.node_num === nodeNum);
    
  const freeCPUs = cpusInNode.filter(cpu => cpu.secondsToProcess < 1);
  
  // 1. Create an array of CPUs not in use per node.
  // 2. Assign work to free CPUs
  // 3. If all free CPUs have been exceeded, create
  //    frozen threads.

  // Unpack request
  let overflowIndex = 0;
  for(let i = 0; i < cpusRequested.length; i++) {
    // If there's a free CPU, assign it.

    if(i < freeCPUs.length) {
      messages.push({
        "to": `node_${nodeNum}_cpu_${i}`,
        "type": "cpu_request",
        "data": {
          "user": user,
          "secondsToProcess": cpusRequested[i]["secondsToProcess"]
        }
      });
    } else {
        // Handle OVERFLOW.

        const x = context.neighbors()
                          .filter(n => n.node_num === nodeNum)

        const y = Math.max(...(x.map(c => c.position[2])));

        const topZ = y > globals.cpu.cpus_per_node ? y : globals.cpu.cpus_per_node;

        topPos = [cpusInNode[0].position[0], cpusInNode[0].position[1], topZ + overflowIndex]
        const waitingProcName = createWaitingProc(nodeNum, topPos, messages, cpusRequested[i].secondsToProcess, user);

        overflowIndex++;
      }
  }
}

function createWaitingProc(nodeNum, pos, messages, secondsToProcess, user) {
  const name = `${nodeNum}_waiting_proc`;
  messages.push({
    "to": `hash`,
    "type": "create_agent",
    "data": {
      "agent_name": name,
      "type": "waiting_proc",
      "behaviors": ["waiting_proc.js"],
      "position": pos,
      "color": "purple",
      "node_num": nodeNum,
      "search_radius": 300,
      "data": {
        "user": user,
        "secondsToProcess": secondsToProcess,
      }
    }
  });
  return name;
}

create_cpus.js

/**
 * @param {AgentState} state
 * @param {AgentContext} context
 */
const behavior = (state, context) => {
  
  // Creation should only run once.
  if(context.currentStep != 1) { return state; }
  
  const globals = context.globals();
  let messages = state.get("messages");

  const PLACEMENT_FROM_CENTER = 12;
  let agents = state.get("agents") || [];


  for(node_i = 0; node_i < globals["cpu"]["nodes"]; node_i++){
    for(cpu_i = 0; cpu_i < globals["cpu"]["cpus_per_node"]; cpu_i++) {
      
      // Calculate starting position.
      let y = node_i % 2 == 0 ? (node_i * -1) : node_i;
      y = y % 2 != 0 ? y + 1 : y;
      const x = PLACEMENT_FROM_CENTER;
      const z = cpu_i;

      // Define agent.
      const cpu_agent = {
          "agent_name": `node_${node_i}_cpu_${cpu_i}`,
          "node_num": node_i,
          "cpu_num": cpu_i,
          "behaviors": ["cpu.js"],
          "position": [x, y, z],
          "workingRequestFrom": null,
          "color": globals["cpu"]["free"],
          "secondsToProcess": 0
      }
      agents.push(cpu_agent);

      // Prepare Hash create agent commands.
      messages.push(
        {
          "to": "hash",
          "type": "create_agent",
          "data": cpu_agent
        }
      )
    }
  }

  // Request Hash create CPUs.
  state.set("messages", messages)
  
  // Store agents.
  state.set("agents", agents)
  
  console.log("Created CPUs.")

  return state;
};

user.js

/**
 * @param {AgentState} state
 * @param {AgentContext} context
 */
const behavior = (state, context) => {
  // console.log("USER #######")
  const responseMsgs = context.messages().filter(m => m["type"] === "completed_request");
  const globals = context.globals();

  // Process responses first.
  for(i = 0; i < responseMsgs.length; i++) {
    const response = responseMsgs[i];
    console.log(`RX: ${response.to[0]}`);
    let numActiveRequests = state.get("numActiveRequests");
    numActiveRequests--;
    if(numActiveRequests === 0) { state.set("waiting", false); }
    state.set("numActiveRequests", numActiveRequests);
  }

  // Create a request.
  if(!state.get("waiting")){ 
    state.set("color", "green");
    const numberOfCPUs = randomInt(1, globals["request"]["MAX_QUERIES_PER_REQUEST"]);
    const maxProcessingTime = globals["request"]["MAX_PROCESSING_SECONDS"];

    requestCPUs(state, numberOfCPUs, maxProcessingTime);

    state.set("waiting", true);
  } else {
    state.set("color", "red");
  }

  // console.log("END USER #######")
  return state;
};

function requestCPUs(state, number, maxProcessingTime) {
  const agentName = state.get("agent_name");
  
  const connectionName = `${agentName}_to_lb`;

  const cpuRequests = [];
  for(i = 1; i < number + 1; i++) {
    cpuRequests.push({
      "secondsToProcess": randomInt(0, maxProcessingTime)
    });
  }

  console.log(`TX: ${agentName}`);
  const request = {
    "user": agentName, 
    "request": cpuRequests
  };
  state.set("color", "red");
  state.set("numActiveRequests", cpuRequests.length);
  state.addMessage(connectionName, "cpu_request", request);
}

function randomInt(min, max) {
  return Math.floor(Math.random() * (max - min + 1)) + min;
}

connection.js

/**
 * @param {AgentState} state
 * @param {AgentContext} context
 */
const behavior = (state, context) => {
  // console.log("CONNECTION #######")
  const filter = state.get("cpu_request");

  let msgs = [];
  msgs = context.messages().filter(m => m.to[0].includes("_to_"));
  let scale = state.get("scale");

  if (msgs.length > 0) {
    
    // Forward CPU requests.
    const messages = forwardCPURequests(state, msgs, context);
    state.set("messages", messages);

    state.set("color", "pink");
    scale[0] = 0.5;
    scale[2] = 0.5;
  } else {
    state.set("color", "blue");
    scale[0] = 0.05;
    scale[2] = 0.05;
  }

  state.set("scale", scale);
  // console.log("END CONNECTION #######")
  return state;
};


function forwardCPURequests(state, msgs, context) {
 
  const msgsToLB = msgs.filter(m => m.to[0].includes("to_lb"));
  const msgsToNode = msgs.filter(m => m.to[0].includes("to_node"));

  let messages = [];

  // Forward CPU requests to load-balancer.
  for(i = 0; i < msgsToLB.length; i++){
    const m = msgsToLB[i];
    messages.push({
      "to": "load_balancer",
      "type": "cpu_request",
      "data": m["data"]
    });
  }

  // Forward CPU requests to nodes.
  for(i = 0; i < msgsToNode.length; i++) {
    const m = msgsToNode[i];
    messages = distributeReqOnNode(state, m, messages, context.globals()["cpu"]["cpus_per_node"]);
  }

  return messages;
}

function distributeReqOnNode(state, cpuRequest, messages, maxCPUs) {
  const agentName = state.get("agent_name");
  const nodeNum = agentName.split("lb_to_")[1];
  
  // const cpusRequested = cpuRequest["data"]["load"];
  const requesterName = cpuRequest["data"]["user"];
  const cpusRequested = cpuRequest["data"]["request"];
  

  // 1. Create an array of CPUs not in use per node.
  // 2. Assign work to free CPUs
  // 3. If all free CPUs have been exceeded, create
  //    frozen threads.

  const freeCPUs = state.get("agents");
  // Unpack request
  for(i = 0; i < cpusRequested.length; i++) {
  //   if(i >= )
    messages.push({
      "to": `${nodeNum}_cpu_${i}`,
      "type": "cpu_request",
      "data": {
        "from": requesterName,
        "secondsToProcess": cpusRequested[i]["secondsToProcess"]
      }
    });
  }
  return messages;
}

function createFrozenThread(nodeNum, index) {

}

cpu.js

/**
 * @param {AgentState} state
 * @param {AgentContext} context
 */
const behavior = (state, context) => {
  // console.log("CPU #######")
  const cpuRequests = context.messages();

  // 1. Check if free.
  // 2. Start processing
  // If in use,  
  // 1. Decrement seconds.
  // 2. If seconds == 0, set CPU = free
  if(state.get("secondsToProcess") < 0) {
    initRequest(state, cpuRequests);
  } else {
    processRequest(state);
  }

  // Handle CPUs.
  // console.log("END CPU #######")
  return state;
};


function initRequest(state, cpuRequests) {
  for(i = 0; i < cpuRequests.length; i++){
    const request = cpuRequests[i]; 
    const user = request.data.user;

    state.set("color", "red");
    state.set("user", user);
    state.set("secondsToProcess", request.data.secondsToProcess);

    // If it is a waiting process, 
    // remove the waiting bin.
    if(request.data.waiting_id) {
      state.set("messages", [{
        "to": "hash",
        "type": "remove_agent",
        "data": { "agent_id": request.data.waiting_id }
      }])
    }
  }
  if(cpuRequests.length > 1) { console.log("TOO MANY"); }
}

function processRequest(state) {
  let messages = [];
  const secondsToProcess = state.get("secondsToProcess") - 1;
  const user = state.get("user");
  if(secondsToProcess < 1 && user) { 
    // Return response to user.
    messages.push({
      "to": user,
      "type": "completed_request"
    });
    state.set("color", "green");
  }
  state.set("secondsToProcess", secondsToProcess);
  state.set("messages", messages);
}

lb.js

/**
 * @param {AgentState} state
 * @param {AgentContext} context
 */
const behavior = (state, context) => {
  // console.log("LB #######")
  const globals = context.globals();
  const cpuRequests = context.messages()
                           .filter(message => 
                             message["type"] == "cpu_request");

  let color = state.get("color");
  if(cpuRequests.length > 0){ 
    assignRequest(state, cpuRequests, globals["cpu"]["nodes"]);
    color = "red";
  } else {
    color = "green";
  }
  state.set("color", color);
  // console.log("END LB #######")
};

function assignRequest(state, cpuRequests, numNodes) {
  let assignmentIndex = state.get("assignment_index");
  let messages = [];
  
  for(i = 0; i < cpuRequests.length; i++) {
    const request = cpuRequests[i];
    if(assignmentIndex > numNodes - 1){ assignmentIndex = 0; }
    messages.push({
      "to": `lb_to_node_${assignmentIndex}`,
      "type": "cpu_request",
      "data": request["data"]
    });
    assignmentIndex++;
    state.set("assignment_index", assignmentIndex);
  }

  state.set("messages", messages);

  return state;
}

waiting_proc.js

/**
 * @param {AgentState} state
 * @param {AgentContext} context
 */
const behavior = (state, context) => {
  const incmessages = context.messages().filter(m => m.to[0] === state.get("agent_name"));
  let messages = [];

  const cpusInNode = context.neighbors()
                          .filter(n => n.node_num === state.get("node_num"));
  const freeCPUs = cpusInNode.filter(cpu => cpu.secondsToProcess < 0);
  let data = state.get("data");
  data.waiting_id = state.get("agent_id");
  if(freeCPUs.length > 0) {
    const name = freeCPUs[0].agent_name
    messages.push({
      "to": name,
      "type": "cpu_request",
      "data": data
    });
  }

  // Remove waiting process.
  state.set("messages", messages);
};