Is there a way in Json.NET serialization to distinguish between “null because not present” and “null because null”?

If you use Json.Net’s LINQ-to-JSON API (JTokens, JObjects, etc.) to parse the JSON, you can tell the difference between a null value and a field that simply doesn’t exist in the JSON. For example:

JToken root = JToken.Parse(json);

JToken nested = root["nested"];
if (nested != null)
{
    if (nested.Type == JTokenType.Null)
    {
        Console.WriteLine("nested is set to null");
    }
    else
    {
        Console.WriteLine("nested has a value: " + nested.ToString());
    }
}
else
{
    Console.WriteLine("nested does not exist");
}

Fiddle: https://dotnetfiddle.net/VJO7ay

UPDATE

If you’re deserializing into concrete objects using Web API, you can still use the above concept by creating a custom JsonConverter to handle your DTOs. The catch is that there needs to be a place on your DTOs to store the field status during deserialization. I would suggest using a dictionary-based scheme like this:

enum FieldDeserializationStatus { WasNotPresent, WasSetToNull, HasValue }

interface IHasFieldStatus
{
    Dictionary<string, FieldDeserializationStatus> FieldStatus { get; set; }
}

class FooDTO : IHasFieldStatus
{
    public string Field1 { get; set; }
    public BarDTO Nested { get; set; }
    public Dictionary<string, FieldDeserializationStatus> FieldStatus { get; set; }
}

class BarDTO : IHasFieldStatus
{
    public int Num { get; set; }
    public string Str { get; set; }
    public bool Bool { get; set; }
    public decimal Dec { get; set; }
    public Dictionary<string, FieldDeserializationStatus> FieldStatus { get; set; }
}

The custom converter would then use above LINQ-to-JSON technique to read the JSON for the object being deserialized. For each field in the target object, it would add an item to that object’s FieldStatus dictionary indicating whether the field had a value, was explicitly set to null or did not exist in the JSON. Here is what the code might look like:

class DtoConverter : JsonConverter
{
    public override bool CanConvert(Type objectType)
    {
        return (objectType.IsClass && 
                objectType.GetInterfaces().Any(i => i == typeof(IHasFieldStatus)));
    }

    public override object ReadJson(JsonReader reader, Type objectType, object existingValue, JsonSerializer serializer)
    {
        var jsonObj = JObject.Load(reader);
        var targetObj = (IHasFieldStatus)Activator.CreateInstance(objectType);

        var dict = new Dictionary<string, FieldDeserializationStatus>();
        targetObj.FieldStatus = dict;

        foreach (PropertyInfo prop in objectType.GetProperties())
        {
            if (prop.CanWrite && prop.Name != "FieldStatus")
            {
                JToken value;
                if (jsonObj.TryGetValue(prop.Name, StringComparison.OrdinalIgnoreCase, out value))
                {
                    if (value.Type == JTokenType.Null)
                    {
                        dict.Add(prop.Name, FieldDeserializationStatus.WasSetToNull);
                    }
                    else
                    {
                        prop.SetValue(targetObj, value.ToObject(prop.PropertyType, serializer));
                        dict.Add(prop.Name, FieldDeserializationStatus.HasValue);
                    }
                }
                else
                {
                    dict.Add(prop.Name, FieldDeserializationStatus.WasNotPresent);
                }
            }
        }

        return targetObj;
    }

    public override bool CanWrite
    {
        get { return false; }
    }

    public override void WriteJson(JsonWriter writer, object value, JsonSerializer serializer)
    {
        throw new NotImplementedException();
    }
}

The above converter will work on any object that implements the IHasFieldStatus interface. (Note that you do not need to implement the WriteJson method in the converter unless you intend to do something custom on serialization as well. Since CanWrite returns false, the converter will not be used during serialization.)

Now, to use the converter in Web API, you need to insert it into the configuration. Add this to your Application_Start() method:

var config = GlobalConfiguration.Configuration;
var jsonSettings = config.Formatters.JsonFormatter.SerializerSettings;
jsonSettings.C‌​onverters.Add(new DtoConverter());

If you prefer, you can decorate each DTO with a [JsonConverter] attribute like this instead of setting the converter in the global config:

[JsonConverter(typeof(DtoConverter))]
class FooDTO : IHasFieldStatus
{
    ...
}

With the converter infrastructure in place, you can then interrogate the FieldStatus dictionary on the DTO after deserialization to see what happened for any particular field. Here is a full demo (console app):

public class Program
{
    public static void Main()
    {
        ParseAndDump("First run", @"{
            ""field1"": ""my field 1"",
            ""nested"": {
                ""num"": null,
                ""str"": ""blah"",
                ""dec"": 3.14
            }
        }");

        ParseAndDump("Second run", @"{
            ""field1"": ""new field value""
        }");

        ParseAndDump("Third run", @"{
            ""nested"": null
        }");
    }

    private static void ParseAndDump(string comment, string json)
    {
        Console.WriteLine("--- " + comment + " ---");

        JsonSerializerSettings settings = new JsonSerializerSettings();
        settings.Converters.Add(new DtoConverter());

        FooDTO foo = JsonConvert.DeserializeObject<FooDTO>(json, settings);

        Dump(foo, "");

        Console.WriteLine();
    }

    private static void Dump(IHasFieldStatus dto, string indent)
    {
        foreach (PropertyInfo prop in dto.GetType().GetProperties())
        {
            if (prop.Name == "FieldStatus") continue;

            Console.Write(indent + prop.Name + ": ");
            object val = prop.GetValue(dto);
            if (val is IHasFieldStatus)
            {
                Console.WriteLine();
                Dump((IHasFieldStatus)val, "  ");
            }
            else
            {
                FieldDeserializationStatus status = dto.FieldStatus[prop.Name];
                if (val != null) 
                    Console.Write(val.ToString() + " ");
                if (status != FieldDeserializationStatus.HasValue)
                    Console.Write("(" + status + ")");
                Console.WriteLine();
            }
        }
    }   
}

Output:

--- First run ---
Field1: my field 1 
Nested: 
  Num: 0 (WasSetToNull)
  Str: blah 
  Bool: False (WasNotPresent)
  Dec: 3.14 

--- Second run ---
Field1: new field value 
Nested: (WasNotPresent)

--- Third run ---
Field1: (WasNotPresent)
Nested: (WasSetToNull)

Fiddle: https://dotnetfiddle.net/xyKrg2