SketchMuse

Ever since my hand gesture Processing project, I've been interested in using Processing [1] for various multimedia purposes. The opportunity came up in the electronic music composition [2] I'm taking to create a sequencer in Processing to control audio produced with ChucK [3]. OSC seemed the easiest protocol to enable communication, so I set to work.

The Processing half is named sketch because it allows the user to draw onto the grid arbitrarily; a time cursor continually loops from left to right, transmitting note on/off events over OSC as the cursor hits dark pixels on one of the 16 tracks.

The ChucK half is named muse and "makes noise" whenever on/off events are received. In this case, the noise is generated by a bank of 16 sine oscillators at the frequencies on a scale. I'm so creative.

The code follows. It's unnecessarily long; hopefully I will have time to pare it down some later. Both the ChucK and Processing halves work independently, though neither makes sound without the other.

Sketch

import oscP5.*;
import netP5.*;

// OSC constants
String myConnectPattern = "/server/connect";
String myDisconnectPattern = "/server/disconnect";

/* port the server is listening for incoming messages */
int myListeningPort = 32000;

/* port the clients should listen for incoming messages from the server */
int myBroadcastPort = 12000;

// Drawing constants
int num_steps = 16;
int num_beats = 16;
int line_width = 5;
int pad_width = 15;
int cursor_width = 10;
int draw_left = 5, draw_right, draw_top = 50, draw_bottom;
int draw_width, draw_height;
color draw_color = color(0, 0, 128);
color draw_background = color(220, 220, 220);

// Timing constants
int ticks_per_beat = 40;

// Button colors
color button_up = color(0, 0, 0);
color button_down = color(255, 0, 0);

/* end configuration */

// OSC
OscP5 oscP5;
NetAddressList myNetAddressList = new NetAddressList();

// Timing
float tick = 0;
float tick_rate = 7;

// Drawing
PImage draw_buffer;

// Sound & Controls
boolean[] note_states = new boolean[num_steps];
boolean button_state = false;

/* end variables */

void setup() {
  // Calculate the size of the drawing area
  int width = num_beats * ticks_per_beat + 10;
  draw_bottom = 5 + draw_top +
      num_steps * (line_width + pad_width) + pad_width;
  draw_right = width - 5;
  draw_width = draw_right - draw_left;
  draw_height = draw_bottom - draw_top;

  // Size the window
  size(width, draw_top + draw_height + 5);

  // Cap framerate
  frameRate(30);

  // Initialize & clear the drawing buffer
  draw_buffer = new PImage(draw_width, draw_height);
  for(int i = 0; i < draw_buffer.pixels.length; i++)
    draw_buffer.pixels[i] = draw_background;

  // Start OSC server
  oscP5 = new OscP5(this, myListeningPort);

  // Initialize the note states
  for(int i = 0; i < num_steps; i++)
    note_states[i] = false;
}

void draw() {
  tick = (tick + tick_rate) % draw_width; // TODO: use real time

  // Update note states
  for(int i = 0; i < num_steps; i++) {
    boolean noteon = false;
    for(int p = 0; p < line_width; p++) {
      int p = pix((int) tick,
          (line_width + pad_width) * i + pad_width + p, draw_width);
      if(draw_buffer.pixels[p] == draw_color) {
        noteon = true;
        break;
      }
    }
    if(noteon != note_states[i])
      oscEvent(new OscMessage("/note",
          new Object[] { new Integer(noteon ? 1 : 0), new Integer(i) }));
    note_states[i] = noteon;
  }

  // Clear the background
  background(0, 0, 0);

  // Place the draw buffer
  copy(draw_buffer,
       0, 0, draw_width, draw_height,
       draw_left, draw_top, draw_width, draw_height);

  // Draw the button
  noStroke();
  fill(button_state ? button_down : button_up);
  rect(0, 0, width, draw_top);

  // Cover the areas in-between steps
  fill(color(255, 255, 255));
  noStroke();
  for(int i = 0; i < num_steps + 1; i++) {
    int y = draw_top + (line_width + pad_width) * i;
    rect(draw_left, y, draw_width, pad_width);
  }

  // Draw the beat lines
  stroke(180, 180, 180);
  for(int i = 0; i < draw_width; i += ticks_per_beat)
    line(draw_left + i, draw_top, draw_left + i, draw_bottom);

  // Draw the time cursor
  stroke(255, 0, 0);
  line(draw_left + (int) tick, 0, (int) draw_left + tick, height);

  // Draw the mouse cursor
  if(mouseX >= draw_left && mouseX <= draw_right &&
     mouseY >= draw_top && mouseY <= draw_bottom) {
    stroke(0, 0, 0);
    noFill();
    rect(mouseX, mouseY - cursor_width,
        cursor_width, cursor_width*2);
  }
}

void mouseDragged() {
  if(mouseX >= draw_left && mouseX <= draw_right &&
     mouseY >= draw_top && mouseY <= draw_bottom) {
    color c = mouseButton == LEFT ? draw_color : draw_background;
    for(int x = mouseX-draw_left; x < mouseX-draw_left+cursor_width; x++) {
      for(int y = mouseY-draw_top-cursor_width; y < mouseY-draw_top+cursor_width; y++) {
        if(x >= 0 && x < draw_width && y >= 0 && y < draw_height)
          draw_buffer.pixels[pix(x, y, draw_width)] = c;
      }
    }
  }
}

void mousePressed() {
  if(mouseY < draw_top) {
    button_state = !button_state;
    oscEvent(new OscMessage("/button",
        new Integer[] { new Integer(button_state ? 1 : 0) }));
  }

  mouseDragged();
}

void keyPressed() {
  if(key == ' ')
    cursor_width = (cursor_width == 10 ? 30 : 10);
  else if(keyCode == UP)
    tick_rate *= 1.2;
  else if(keyCode == DOWN)
    tick_rate *= 0.8333;
  else if(keyCode == LEFT) {
    /* shift left */
  }

  if(tick < 0.01) tick = 0.01;
}

color pix(int x, int y, int width) {
  return x + y*width;
}

/* OSC stuff */

void oscEvent(OscMessage theOscMessage) {
  /* check if the address pattern fits any of our patterns */
  if (theOscMessage.addrPattern().equals(myConnectPattern)) {
    connect(theOscMessage.netAddress().address());
  } else if (theOscMessage.addrPattern().equals(myDisconnectPattern)) {
    disconnect(theOscMessage.netAddress().address());
  } else {
    /* if pattern matching was not successful, then
     * broadcast the incoming message to all addresses
     * in the netAddresList. 
     */
    oscP5.send(theOscMessage, myNetAddressList);
  }
}

private void connect(String theIPaddress) {
  if (!myNetAddressList.contains(theIPaddress, myBroadcastPort)) {
    myNetAddressList.add(new NetAddress(theIPaddress, myBroadcastPort));
    println("### adding "+theIPaddress+" to the list.");
  } else {
    println("### "+theIPaddress+" is already connected.");
  }
  println("### "+myNetAddressList.list().size()+" clients connected.");
}

private void disconnect(String theIPaddress) {
  if (myNetAddressList.contains(theIPaddress, myBroadcastPort)) {
    myNetAddressList.remove(theIPaddress, myBroadcastPort);
    println("### removing "+theIPaddress+" from the list.");
  } else {
    println("### "+theIPaddress+" is not connected.");
  }
  println("### currently there are "+myNetAddressList.list().size());
}

Muse

"localhost" => string host;
32000 => int port;
OscSend xmit;
xmit.setHost(host, port);
xmit.startMsg("/server/connect", "");

OscRecv recv;
12000 => recv.port;
recv.listen();
recv.event( "/note, i i" ) @=> OscEvent noteup;
recv.event( "/button, i" ) @=> OscEvent buttonup;

// the patch
SinOsc s[16];
Envelope env[16];
for(0 => int i; i < s.cap(); i++) {
    Std.mtof((15-i) * 1 + 60) => s[i].freq;
    .2 => s[i].gain;
    s[i] => env[i] => dac;
}

fun void note_waiter() {
    while(true) {
        noteup => now;
        while(noteup.nextMsg())
        {
            int state, note;

            noteup.getInt() => state;
            noteup.getInt() => note;
            if(state == 1)
                env[note].keyOn(1);
            else
                env[note].keyOff();
            <<< state, note >>>;
        }
    }
}

fun void button_waiter() {
    while(true) {
        buttonup => now;
        while(buttonup.nextMsg())
        {
            int state;
            buttonup.getInt() => state;
            if(state == 1) {
            } else {
            }
        }
    }
}

spork ~ note_waiter();
spork ~ button_waiter();
while(1::second => now);