For those that remember the film Minority Report, you will probably recall the way Tom Cruise’s character interacted with the digital display – using hand motions to carry out different actions. Now, that sort of thing is easily possible, thanks to a new $70 device called the Leap Motion.

The Leap Motion controller is a tiny device that captures (in real-time), the individual finger and hand movements of a user. What this means for a developer is they can create applications that are controlled simply by gesturing with their hands/fingers, just like in the Minority Report film!

I recently received my developer preview of the Leap Motion device, and once connected to my Mac via USB cable, I installed the software and opened up some of the sample code. One of the samples was a simple HTML page, and this demonstrated how data from the Leap device could be read in JSON format – by using a WebSocket connection that streams finger/hand coordinate information to the web page. Here is a sample of the JSON returned:

{
  "hands": [
    {
      "fingers": [
        {
          "id": 8,
          "length": 54.496,
          "tip": {
            "direction": [
              0.0687786,
              -0.260609,
              -0.962992
            ],
            "position": [
              -9.31378,
              254.178,
              110.934
            ],
            "velocity": [
              27.4984,
              41.4125,
              896.409
            ]
          },
          "tool": false,
          "width": 12.7935
        }
      ],
      "id": 1,
      "palm": {
        "ball": {
          "center": [
            -3.7402,
            259.651,
            121.732
          ],
          "radius": 32.3772
        },
        "direction": [
          0.0397018,
          -0.0721033,
          -0.855305
        ],
        "normal": [
          0.390255,
          -0.692151,
          0.511843
        ],
        "position": [
          -2.43423,
          258.702,
          128.253
        ],
        "velocity": [
          -124.89,
          6.94538,
          310.019
        ]
      }
    }
  ],
  "id": 372792,
  "timestamp": 36043508229
}

As you can see above, there is a wealth of information available to the developer, including direction, position and velocity data for each finger.

After seeing the power of the device, I wanted to see how this could work with a Sencha Touch application, by basically allowing the user to use their finger to navigate through a Sencha Touch app.

First, I defined a new class in my sample Sencha Touch project, called “Ext.interaction.LeapMotion”, and after a few evenings working on the code I eventually ended up with the following:

Ext.define('Ext.interaction.LeapMotion', {
    config: {
	webSocket: null,
	previousScroll: 0,
	pressed: false,
	previouslyPressed: false,
	previousElement: null,
	scroll: 0,
	pointer: null,	// The pointer drawn on screen
	movementThreshold: 40,	// Movement threshold for mousedown events
	
	// The coordinates of the Leap Motion to work with
	centerX: 0,
	minX: -125,
	maxX: 125,
	centerY: 300,
	minY: 200,
	maxY: 400,
	
	eventFiredX: 0,
	eventFiredY: 0,
	
	// Coordinates for centre of the page
	pageCenterX: (document.body.clientWidth / 2),
	pageCenterY: (document.body.clientHeight / 2),
	
	maxPageX: (document.body.clientWidth - 40),
	minPageX: 0,
	minPageY: 0,
	maxPageY: (document.body.clientHeight - 40),
	
	ratioX: 0,
	ratioY: 0,
	newX: 0,
	newY: 0
    },
	
    constructor: function(config) {
        this.initConfig(config);
        
        this.setRatioX(this.getPageCenterX() / this.getMaxX());
        this.setRatioY(this.getMaxPageY() / (this.getMaxY() - this.getMinY()));
				        
        this.createPointer();
        this.createWebSocket();
    },
    
    createWebSocket: function() {
    	this.setWebSocket(new WebSocket("ws://localhost:6437/"));
    	
    	var me = this,    	
    		webSocket = this.getWebSocket();  
    	  	
    	webSocket.onmessage = function(event) {
			var obj = JSON.parse(event.data);
				
			if (obj.hands.length > 0 && obj.hands[0].fingers.length > 0)
			{			
				// Leap Motion device coordinates
				var x = obj.hands[0].fingers[0].tip.position[0],
					y = obj.hands[0].fingers[0].tip.position[1];
					
				me.setPressed(obj.hands[0].fingers[0].tip.position[2] < 0);		// Is the user motioning towards the screen, if so, fire events on the underlying element.
	
				me.getPointer().getSurface("main")._items.items[0].repaint();
			
				if (x > 0)
					me.setNewX((x * me.getRatioX()) + me.getPageCenterX());
				else
					me.setNewX(me.getPageCenterX() - (x * (me.getRatioX() * -1)));
	
				me.setNewY((((y - 300) * me.getRatioY()) * -1) + 300);
							
				if (me.getNewY() < 0)
					me.setNewY(0);
				else if (me.getNewY() > me.getMaxPageY())
					me.setNewY(me.getMaxPageY());
					
				if (me.getNewX() < 0)
					me.setNewX(0);
				else if (me.getNewX() > me.getMaxPageX())
					me.setNewX(me.getMaxPageX());
	
				me.getPointer().setLeft(me.getNewX());
				me.getPointer().setTop(me.getNewY());
	
				if (me.getPressed())
				{
					if (me.getPreviousElement() != null)
					{
						var diffX = 0;
						var diffY = 0;
						
						if (me.getEventFiredX() < me.getNewX())
							diffX = me.getNewX() - me.getEventFiredX();
						else
							diffX = me.getEventFiredX() - me.getNewX();
							
						if (me.getEventFiredY() < me.getNewY())
							diffY = me.getNewY() - me.getEventFiredY();
						else
							diffY = me.getEventFiredY() - me.getNewY();
						
						if (diffX > me.getMovementThreshold() || diffY > me.getMovementThreshold()) {
							if (Ext.os.is.iOS)
								me.firePageEvent(me.getPreviousElement(), 'touchmove');
							else
								me.firePageEvent(me.getPreviousElement(), 'mousemove');
						}
					}
				
					me.getPointer().getSurface("main")._items.items[0].setAttributes({ fillStyle: "red" });
				}
				else
				{
					me.getPointer().getSurface("main")._items.items[0].setAttributes({ fillStyle: "green" });
					me.setPreviouslyPressed(false);
					
					if (me.getPreviousElement() != null)
					{
						if (Ext.os.is.iOS)
							me.firePageEvent(me.getPreviousElement(), 'touchend');
						else
							me.firePageEvent(me.getPreviousElement(), 'mouseup');
					}
				}
	
				if (me.getPressed() && me.getPreviouslyPressed() === false)
				{
					var element = me.getComponentFromPosition(me.getNewX() - 1, me.getNewY() - 1);
				
					if (Ext.os.is.iOS)
						me.firePageEvent(element, "touchstart");
					else
						me.firePageEvent(element, "mousedown");
													
					me.setPreviouslyPressed(true);
					me.setPreviousElement(element);
					me.setEventFiredX(me.getNewX());
					me.setEventFiredY(me.getNewY());
				}
			}
		};
    	
    		webSocket.onclose = function(event) {
			ws = null;
		};
		
		webSocket.onerror = function(event) {
			alert("Received error");
		};
    },
    
    firePageEvent: function(element, eventName) {
    	if (Ext.os.is.iOS) {
    	 	var touch = document.createTouch(window, element, null, this.getNewX(), this.getNewY(), this.getNewX(), this.getNewY());
		var touches = document.createTouchList(touch);
		var targetTouches = document.createTouchList(touch);
		var changedTouches = document.createTouchList(touch);
			    
		var evt = document.createEvent("TouchEvent");
		evt.initTouchEvent(eventName, true, true, window, null, 0, 0, 0, 0, false, false, false, false, touches, targetTouches, changedTouches, 1, 0);
		var event = element.dispatchEvent(evt);
       	}
       	else {
       		var evt = document.createEvent("MouseEvents");
		evt.initMouseEvent(eventName, true, true, null, null, null, null, this.getNewX(), this.getNewY());
		var event = element.dispatchEvent(evt);
       	}
    },
    
    getComponentFromPosition: function(x, y) {
    	var el = document.elementFromPoint(x, y);
   
   	return el;
    },    
        
    createPointer: function() {
    	this.setPointer(new Ext.draw.Component({
    		style: 'position: absolute',
      		zIndex: 2000,
      		top: -100,
      		left: -100,
      		width: 40,
      		height: 40,
		  	items: [{
		    	type: 'circle',
		    	cx: 20,
		    	cy: 20,
		    	r: 20,
		    	fillStyle: 'green'
		  	}]
		}));
		
	Ext.Viewport.add(this.getPointer());
    }
});

This class can then be used by requiring it in a project:

requires: [
    "Ext.interaction.LeapMotion"
]

Then instantiate it at the launch of the app:

launch: function() {
    Ext.create("Ext.interaction.LeapMotion");
}

The code is quite crude at the moment, but it works with limited functionality. This could potentially be extended to take in to account more than one finger, therefore also replicating pinch to zoom events. However, a demonstration of this working so far can be seen in the following YouTube video: