This section describes how to design a scene to be used with the object tracker.

Live Video Input

images/download/attachments/125454677/liveinput.png

The most important setting of a scene that can be used with the Object Tracker is the live input. The SHM Aux Mode must be set to Send. This allows the Object Tracker to read the input surface from the Engine. The Live Input asset needs to be present in the scene but does not necessarily need to be visible, so the result can be still composed on an external mixer.

It is possible to use other texture source such as clip channels. You need to make sure that the SHM Aux Mode is set to Send and that the configured Input Key (see Configuration > Tracking in the Viz Arc User Guide) matches the SHMAuxKey configuration of the Object Tracker's Viz Engine (for example, viz_clip1_aux, viz_live1_aux, etc.).

Script

images/download/attachments/125454677/CaptureWindow.PNG

Tracking Script

As the tracking data comes into Viz Engine through shared memory, these values need to be read and translated into screen coordinates. This can happen through the following script:

Dim trackerID = 0
Dim screenWidth as Double
Dim screenHeight as Double
Dim mpi = 3.14159265359
Dim degToRad = mpi / 180.0
Dim currentTrackingType as String
	
Structure tracked
	x as Double
	y as Double
	width as Double
	height as Double
	valid as Integer
	score  as Double
End Structure
 
Structure trackedObjects
	'TIMECODE
	tc as Integer
	'TRACKED OBJECTS
	tos as Array[tracked]
End Structure
 
Structure Point3D
	x as Double
	y as Double
	z as Double
End Structure
 
Structure face
	valid as Integer
	Outline as Array[Point3D]
	EyeLeft as Array[Point3D]
	EyeRight as Array[Point3D]
	Mouth as Array[Point3D]
	BetweenEyes as Point3D
	EyeCornerLeft as Point3D
	EyeCornerRight as Point3D
	PupilLeft as Point3D
	PupilRight as Point3D
	NoseTip as Point3D
	CheekLeft as Point3D
	CheekRight as Point3D
	MouthCornerLeft as Point3D
	MouthCornerRight as Point3D
	Chin as Point3D
	Roll as Double
	Pitch as Double
	Yaw as Double
End Structure
 
Structure faceObjects
	'TIMECODE
	tc as Integer
	'FACE OBJECTS
	fs as Array[face]
End Structure
 
Structure pose
	valid as Integer
	skeleton as Array[Point3D]
End Structure
 
Structure poseObjects
	'TIMECODE
	tc as Integer 
	'POSE OBJECTS
	ps as Array[pose]
End Structure
 
Sub OnInit()
	'do some trigonometry to get the screen width/height on the 0 plane.
	'this calculation works in orto and perspectiv mode but the camera needs
	'to look straight down the z axis (no rotation). Works well with the 
	'default camera settings 
	screenWidth = tan(scene.CurrentCamera.Fovx*0.5*degToRad)*scene.CurrentCamera.Position.z*2
	screenHeight = tan(scene.CurrentCamera.Fovy*0.5*degToRad)*scene.CurrentCamera.Position.z*2
 
	VizCommunication.map.RegisterChangedCallback("FaceObjects")
	VizCommunication.map.RegisterChangedCallback("PoseObjects")
	VizCommunication.map.RegisterChangedCallback("TrackedObjects")
End Sub
 
Dim somethingToShow as Boolean
Dim pos as Vertex
Dim rot as Vertex
Dim scale as Double
Dim bbHeight as Double
Dim bbWidth as Double
Dim quality as Double
 
Sub Move()
	Dim bbTarget     = GetParameterContainer("bbTarget")
	Dim centerTarget = GetParameterContainer("centerTarget")
	Dim gfxTarget    = GetParameterContainer("gfxTarget")
	
	centerTarget.position.x = pos.x
	centerTarget.position.y = pos.y
	
	centerTarget.Rotation.x = rot.x
	centerTarget.Rotation.y = rot.y
	centerTarget.Rotation.z = rot.z
	
	centerTarget.Scaling.x = scale
	centerTarget.Scaling.y = scale
	centerTarget.Scaling.z = scale
	
	bbTarget.position.x = pos.x
	bbTarget.position.y = pos.y
	bbTarget.Scaling.x  = bbWidth  / 100
	bbTarget.Scaling.y  = bbHeight / 100
	
	if GetParameterBool("gfxFollowX") == true Then
		if(GetParameterBool("gfxFollowBBX") == true) then
			gfxTarget.position.X = pos.X + (bbWidth + GetParameterDouble("gfxDeltaX") * screenWidth) * 0.5
		else
 			gfxTarget.position.X = pos.X + GetParameterDouble("gfxDeltaX") * 0.5 * screenWidth
		end if
  	end If
 
  	if GetParameterBool("gfxFollowY") == true Then
		If(GetParameterBool("gfxFollowBBY") == true) then
			gfxTarget.position.Y = pos.Y + (bbHeight + GetParameterDouble("gfxDeltaY") * screenHeight) * 0.5	
		Else
			gfxTarget.position.Y = pos.Y + GetParameterDouble("gfxDeltaY") * 0.5 * screenHeight
		End if
	End If
 
	if GetParameterBool("gfxDoScale") == true Then
		Dim inMin as Double
		Dim inMax as Double
		Dim outMin as Double
		Dim outMax as Double
		Dim gfxScale = bbHeight
	
		inMin = GetParameterDouble("gfxInMinScale")
	   	inMax = GetParameterDouble("gfxInMaxScale")
 
		outMin = GetParameterDouble("gfxOutMinScale")
	   	outMax = GetParameterDouble("gfxOutMaxScale")
 
		gfxScale = Max(inMin, gfxScale)
		gfxScale = Min(inMax, gfxScale)
 
		gfxScale = (gfxScale-inMin)/(inMax-inMin)
		gfxScale = outMin + gfxScale*(outMax-outMin)
 
		gfxTarget.scaling.x = gfxScale
		gfxTarget.scaling.y = gfxScale
		gfxTarget.scaling.z = gfxScale
	End If
 
	Dim tornadoActive as Boolean
	tornadoActive = GetParameterBool("tornadoActive")	
	If tornadoActive Then
		Dim tornadoTarget = GetParameterContainer("tornadoTarget")
		Dim loc = tornadoTarget.WorldPosToLocalPos(pos)
		
		tornadoTarget.GetGeometryPluginInstance().SetParameterDouble("x3", loc.x + GetParameterDouble("tornadoDeltaX") * bbWidth / gfxTarget.scaling.x)
		tornadoTarget.GetGeometryPluginInstance().SetParameterDouble("y3", loc.y + GetParameterDouble("tornadoDeltaY") * bbHeight / gfxTarget.scaling.y)
	End If
End Sub
 
Sub FaceTracking()
	Dim fObjects as faceObjects	
	fObjects = (faceObjects)VizCommunication.Map["FaceObjects"]
 
	If	fObjects.fs.Size > trackerID Then
		somethingToShow = True
	Else
		somethingToShow = False
	End If
	
	If	somethingToShow Then
		somethingToShow = fObjects.fs[trackerID].valid == 1
	End If
	
	If	somethingToShow Then
		rot.x = -fObjects.fs[trackerID].Pitch / degToRad
		rot.y =  fObjects.fs[trackerID].Yaw   / degToRad
		rot.z = -fObjects.fs[trackerID].Roll  / degToRad
 
		pos.x =  fObjects.fs[trackerID].NoseTip.x * screenWidth
		pos.y = -fObjects.fs[trackerID].NoseTip.y * screenHeight
		
		dim cheekLeft as Vertex
		dim cheekRight as Vertex
		
		cheekLeft.x  = fObjects.fs[trackerID].CheekLeft.x  * screenWidth
		cheekLeft.y  = fObjects.fs[trackerID].CheekLeft.y  * screenHeight
		cheekLeft.z  = fObjects.fs[trackerID].CheekLeft.z  * screenWidth
		cheekRight.x = fObjects.fs[trackerID].CheekRight.x * screenWidth
		cheekRight.y = fObjects.fs[trackerID].CheekRight.y * screenHeight
		cheekRight.z = fObjects.fs[trackerID].CheekRight.z * screenWidth	
		
		dim cheek_distance as Double
		cheek_distance = Distance(cheekLeft, cheekRight)
		cheek_distance /= 100		
		
		scale = cheek_distance
		
		Move()
	End If
End Sub 
 
Sub ObjectTracking()   
	Dim trObjects as trackedObjects	
	trObjects = (trackedObjects)VizCommunication.Map["TrackedObjects"]
 
	somethingToShow = trObjects.tos.Size > trackerID
	If (somethingToShow) Then
		somethingToShow = trObjects.tos[trackerID].valid == 1
	End If
 
	If (somethingToShow) Then
		pos.x    =  trObjects.tos[trackerID].x * screenWidth
		pos.y    = -trObjects.tos[trackerID].y * screenHeight
		bbWidth  =  trObjects.tos[trackerID].width  * screenWidth
		bbHeight =  trObjects.tos[trackerID].height * screenHeight
		
		rot.x = 0.0
		rot.y = 0.0
		rot.z = 0.0
		scale = 1.0
 
		Move()
 
		quality  = trObjects.tos[trackerID].score		
		findSubContainer("quality").GetFunctionPluginInstance("ControlNum").SetParameterString("input", cStr(quality*100.0))
    End If
End Sub
 
Sub PoseTracking()
	Dim jointIdx = GetParameterInt("trackingJoint")
	Dim pObjects as poseObjects	
	pObjects = (poseObjects)VizCommunication.Map["PoseObjects"]
	
	somethingToShow = pObjects.ps.Size > trackerID
	If (somethingToShow) Then
		somethingToShow = pObjects.ps[trackerID].valid == 1
	End If	
	
	If (somethingToShow) Then
		pos.x =  pObjects.ps[trackerID].Skeleton[jointIdx].x * screenWidth
		pos.y = -pObjects.ps[trackerID].Skeleton[jointIdx].y * screenHeight
		'pos.z =  pObjects.ps[trackerID].Skeleton[jointIdx].z
		
		rot.x = 0.0
		rot.y = 0.0
		rot.z = 0.0
		scale = 1.0
		bbHeight = 0
		bbWidth = 0
		
		Move()
	End If
End Sub
 
Sub OnExecPerField()
    trackerID = GetParameterInt("trackingID") - 1
 
    Dim trackingActive as Boolean
	trackingActive = GetParameterBool("trackingActive")	
 
	If(trackingActive) Then	
		If( currentTrackingType == "FaceObjects") Then
			FaceTracking()
	    ElseIf( currentTrackingType == "TrackedObjects") Then
			ObjectTracking()		
		ElseIf( currentTrackingType == "PoseObjects") Then
			PoseTracking()
	    End If
	End If
		
	Dim visibilityTarget = GetParameterContainer("visibilityTarget")
 
	visibilityTarget.Active = trackingActive AND somethingToShow
	this.Active = trackingActive AND somethingToShow
End Sub
 
Sub OnInitParameters()
   	RegisterParameterBool("trackingActive", "Tracking Active", True)
	RegisterParameterInt("trackingID", "Tracking ID", 1, 1, 5)
		
	RegisterParameterContainer("gfxTarget", "GFX Target")
	RegisterParameterBool("gfxFollowX", "GFX - Follow X", False)
	RegisterParameterBool("gfxFollowBBX", "GFX - Follow Bounding Box Width", False)
	RegisterParameterDouble("gfxDeltaX", "GFX - Distance X", 0.0, -1000000, 1000000)
	RegisterParameterBool("gfxFollowY", "GFX - Follow Y", False)
	RegisterParameterBool("gfxFollowBBY", "GFX - Follow Bounding Box Height", False)
	RegisterParameterDouble("gfxDeltaY", "GFX - Distance Y", 0.0, -1000000, 1000000)	
 
	RegisterParameterBool("gfxDoScale", "GFX - Scale by Bounding Box", False)
 
	RegisterParameterDouble("gfxInMinScale", "GFX - Input Min Scale", 0.0, 0, 1.0)		
	RegisterParameterDouble("gfxInMaxScale", "GFX - Input Max Scale", 1.0, 0, 1.0)		
 
	RegisterParameterDouble("gfxOutMinScale", "GFX - Output Min Scale", 0.0, 0, 1000000)		
	RegisterParameterDouble("gfxOutMaxScale", "GFX - Output Max Scale", 1.0, 0, 1000000)		
	
	RegisterParameterBool("tornadoActive", "Tornado Active", False)
	RegisterParameterContainer("tornadoTarget", "Tornado - Target")
	RegisterParameterDouble("tornadoDeltaX", "Tornado - Distance X", 0.0, -1000000, 1000000)
	RegisterParameterDouble("tornadoDeltaY", "Tornado - Distance Y", 0.0, -1000000, 1000000)
	
	RegisterParameterInt("trackingJoint", "Tracking Pose Joint", 0, 0, 32)	
	
	RegisterParameterContainer("visibilityTarget", "Visibility Target")	
	RegisterParameterContainer("centerTarget", "Debug - Center Target")
	RegisterParameterContainer("bbTarget", "Debug - BB Target")
End Sub
 
Sub OnSharedMemoryVariableChanged(map As SharedMemory, mapKey As String)
	currentTrackingType = mapKey
End Sub

images/download/attachments/125454677/Capture.PNG

In a first step, the script calculates in the OnInit method the available width and height of the viewport on the zero Z plane. In the OnExecPerField method, the tracking data is read out using a given shared memory keys.

The TrackingID parameter selects the tracked point to follow in the tracking points data structures (it represents also the Tracked Object Input ID in the Viz Arc interface).

The script uses the GFX Target (that represents the tracked point), it is used very much like the Autofollow plug-in. Additionally it uses a few more parameters that determine an additional offset relative to the tracked object's bounding box.

GFX - Follow X: Allows the graphics follow the horizontal movement of the tracked object.
GFX - Follow Bounding Box Width: Positions the graphics on the right hand side of the bounding box.
GFX - Distance X: Adds constant horizontal offset.
GFX - Follow Y: Allows the graphics follow the vertical movement of the tracked object.
GFX - Follow Bounding Box Height: Aligns the graphics on top of the bounding box.
GFX - Distance Y: Adds constant vertical offset.
GFX - Scale by Bounding Box: Uses Bounding Box size to scale graphics.
GFX - Input Min Scale: Sets minimum input scale of the Bounding Box height. The input height is normalized between 0 and 1 (where 1 is the full screen height, 0.1 is 10% of the screen height etc.).
GFX - Input Max Scale: Sets maximum input scale of the Bounding Box height. The input height is normalized between 0 and 1 (where 1 is the full screen height, 0.1 is 10% of the screen height etc.).
GFX - Output Min Scale: Maps minimum output scale of the graphics.
GFX - Output Max Scale: Maps maximum output scale of the graphics.

Using the above sample values, a bounding box with height of 0.05 or smaller (thus 5% of the screen height or smaller) results in a scaling of 0.4 (Output Min Scale). A bounding box of height 0.2 or larger (20% or larger of the screen height) is scaled to 0.7.

All values in between 0.05 and 0.2 are interpolated linearly between 0.4 and 0.7. Adjust those values to suit your graphics. The sample in this script considers only the height of the bounding box, but it could be easily changed to consider the surface of the bounding box or the width only.

The checkbox Tornado Active enables both the visualization and the evaluation of the position of the Tornado - Target.

It can be customized through:

Tornado - Distance X: The horizontal offset from the tracked point.
Tornado - Distance Y: The vertical offset from the tracked point.

The Tracking Pose Joint selects the landmark to be used as tracked point.

The script then switches on and off the target container defined in the Visibility Target parameter when tracking begins and tracking ends or is lost. This can be also replaced with a stage command for example.

The parameter container Debug - Center Target is updated on every field to the "check" the actual tracking position.

Parent Transformations: The script above does not consider any parent transformations of the target container. Make sure the target container contains no additional parent transformations.

The Debug - BB Target container parameter (which might contain a Noggi or Rectangle plug-in) gets resized according to the tracked width and height of the object.

Bounding Box Sizes: The bounding box sizes might become zero. This is always the case for simple and manual tracking.

Data Structures

Detection and Tracking, Simple Tracking and Manual Tracking

Each frame the Object Tracker updates the VizCommunicationMap variable with key TrackedObjects. It contains the structure shown below:

Structure tracked
	x as Double
	y as Double
	width as Double
	height as Double
	valid as Integer
	score  as Double
End Structure
 
Structure trackedObjects
	'TIMECODE
	tc as Integer
	'TRACKED OBJECTS
	tos as Array[tracked]
End Structure

The trackedObjects structure contains an array of five tracked structures (the maximum number of objects that can be tracked); the parameter valid shows whether the object is actually tracked. The position inside the array matches the InputID used when the object was selected.

Face Tracking

Each frame the Object Tracker updates the VizCommunicationMap variable with key FaceObjects. It contains the structure shown below:

Structure Point3D
	x as Double
	y as Double
	z as Double
End Structure
 
Structure face
	valid as Integer
	Outline as Array[Point3D]
	EyeLeft as Array[Point3D]
	EyeRight as Array[Point3D]
	Mouth as Array[Point3D]
	BetweenEyes as Point3D
	EyeCornerLeft as Point3D
	EyeCornerRight as Point3D
	PupilLeft as Point3D
	PupilRight as Point3D
	NoseTip as Point3D
	CheekLeft as Point3D
	CheekRight as Point3D
	MouthCornerLeft as Point3D
	MouthCornerRight as Point3D
	Chin as Point3D
	Roll as Double
	Pitch as Double
	Yaw as Double
End Structure
 
Structure faceObjects
	'TIMECODE
	tc as Integer
	'FACE OBJECTS
	fs as Array[face]
End Structure

The faceObjects structure contains an array of five face structures (the maximum number of objects that can be tracked); the parameter valid shows whether the face is actually tracked. The position inside the array matches the InputID used when the face was selected.

2D Pose Tracking

Each frame the ObjectTracker updates the VizCommunicationMap variable with key PoseObjects. It contains the structure shown below:

Structure Point3D
	x as Double
	y as Double
	z as Double
End Structure
 
Structure pose
	valid as Integer
	skeleton as Array[Point3D]
End Structure
 
Structure poseObjects
	'TIMECODE
	tc as Integer 
	'POSE OBJECTS
	ps as Array[pose]
End Structure

The poseObjects structure contains an array of five pose structures (the maximum number of objects that can be tracked); the parameter valid shows whether the object is actually tracked. The position inside the array matches the InputID used when the person was selected. The skeleton array is filled as follows:

images/download/attachments/125454677/pose_points.png

If a landmark is not detected its coordinates are outside the screen (x:-1.0 y:-1.0 z:-1.0).

Object Tracker User Guide

Viz Scene Design

Live Video Input

Script

Tracking Script

Data Structures

Detection and Tracking, Simple Tracking and Manual Tracking

Face Tracking

2D Pose Tracking

Search results