Found the solution from here: http://social.msdn.microsoft.com/Forums/en-US/wpf/thread/3d0eaa54-09a9-4c51-8677-8e90577e7bac/
XAML:
<UserControl.Resources>
<Style x:Key="itemstyle" TargetType="{x:Type ListViewItem}">
<EventSetter Event="MouseDoubleClick" Handler="HandleDoubleClick" />
</Style>
</UserControl.Resources>
<ListView Name="TrackListView" ItemContainerStyle="{StaticResource itemstyle}">
<ListView.View>
<GridView>
<GridViewColumn Header="Title" Width="100" HeaderTemplate="{StaticResource BlueHeader}" DisplayMemberBinding="{Binding Name}"/>
<GridViewColumn Header="Artist" Width="100" HeaderTemplate="{StaticResource BlueHeader}" DisplayMemberBinding="{Binding Album.Artist.Name}" />
</GridView>
</ListView.View>
</ListView>
C#:
protected void HandleDoubleClick(object sender, MouseButtonEventArgs e)
{
var track = ((ListViewItem) sender).Content as Track; //Casting back to the binded Track
}