Add the possibility to specify a threshold for the number of negative cache

results required to have the cache return lookup failure.

A new configuration parameter is introduced, which must be set to a value
greater than 1 to activate this feature. The default behavior is unchanged.

The purpose of this change is to allow probes for the existence of an entry
(which are expected to fail), before that entry is added to one of the
queried databases, without the cache returning the stale information from
the probe query until that cache entry expires. If, for example, a new user
account is created after checking that the new account name is available,
the negative cache entry would prevent immediate access to the account.

For that example, the new configuration option

negative-confidence-threshold passwd 2

will require a second negative query result to consider the negative cache
entry for a passwd entry valid, but if the user account has been created
between the queries, then the positive query result from the second query
will be cached and returned.

usr.sbin/nscd/cachelib.c

@@ -726,6 +726,12 @@ cache_read(struct cache_entry_ *entry, const char *key, size_t key_size,
 		TRACE_OUT(cache_read);
 		return (-1);
 	}
+	/* pretend that entry was not found if confidence is below threshold*/
+	if (find_res->confidence < 
+	    common_entry->common_params.confidence_threshold) {
+		TRACE_OUT(cache_read);
+		return (-1);
+	}
 
 	if ((common_entry->common_params.max_lifetime.tv_sec != 0) ||
 		(common_entry->common_params.max_lifetime.tv_usec != 0)) {
@@ -826,6 +832,24 @@ cache_write(struct cache_entry_ *entry, const char *key, size_t key_size,
 	item = HASHTABLE_GET_ENTRY(&(common_entry->items), hash);
 	find_res = HASHTABLE_ENTRY_FIND(cache_ht_, item, &item_data);
 	if (find_res != NULL) {
+		if (find_res->confidence < common_entry->common_params.confidence_threshold) {
+		  	/* duplicate entry is no error, if confidence is low */
+			if ((find_res->value_size == value_size) &&
+			    (memcmp(find_res->value, value, value_size) == 0)) {
+				/* increase confidence on exact match (key and values) */
+				find_res->confidence++;
+			} else {
+				/* create new entry with low confidence, if value changed */
+				free(item_data.value);
+				item_data.value = malloc(value_size);
+				assert(item_data.value != NULL);
+				memcpy(item_data.value, value, value_size);
+				item_data.value_size = value_size;
+				find_res->confidence = 1;
+			}
+			TRACE_OUT(cache_write);
+			return (0);
+		}
 		TRACE_OUT(cache_write);
 		return (-1);
 	}
@@ -839,6 +863,8 @@ cache_write(struct cache_entry_ *entry, const char *key, size_t key_size,
 	memcpy(item_data.value, value, value_size);
 	item_data.value_size = value_size;
 
+	item_data.confidence = 1;
+
 	policy_item = common_entry->policies[0]->create_item_func();
 	policy_item->key = item_data.key;
 	policy_item->key_size = item_data.key_size;

usr.sbin/nscd/cachelib.h

@@ -92,6 +92,7 @@ struct common_cache_entry_params {
 	size_t	satisf_elemsize;	/* if entry size is exceeded,
 					 * this number of elements will be left,
 					 * others will be deleted */
+	int	confidence_threshold;	/* number matching replies required */
 	struct timeval	max_lifetime;	/* if 0 then no check is made */
 	enum cache_policy_t policy;	/* policy used for transformations */
 };
@@ -116,6 +117,7 @@ struct cache_ht_item_data_ {
 	size_t	value_size;
 
 	struct cache_policy_item_ *fifo_policy_item;
+	int	confidence;	/* incremented for each verification */
 };
 
 struct cache_ht_item_ {

usr.sbin/nscd/config.c

@@ -209,6 +209,7 @@ create_def_configuration_entry(const char *name)
 	positive_params.max_elemsize = DEFAULT_POSITIVE_ELEMENTS_SIZE;
 	positive_params.satisf_elemsize = DEFAULT_POSITIVE_ELEMENTS_SIZE / 2;
 	positive_params.max_lifetime.tv_sec = DEFAULT_POSITIVE_LIFETIME;
+	positive_params.confidence_threshold = DEFAULT_POSITIVE_CONF_THRESH;
 	positive_params.policy = CPT_LRU;
 
 	memcpy(&negative_params, &positive_params,
@@ -216,6 +217,7 @@ create_def_configuration_entry(const char *name)
 	negative_params.max_elemsize = DEFAULT_NEGATIVE_ELEMENTS_SIZE;
 	negative_params.satisf_elemsize = DEFAULT_NEGATIVE_ELEMENTS_SIZE / 2;
 	negative_params.max_lifetime.tv_sec = DEFAULT_NEGATIVE_LIFETIME;
+	negative_params.confidence_threshold = DEFAULT_NEGATIVE_CONF_THRESH;
 	negative_params.policy = CPT_FIFO;
 
 	memset(&default_common_timeout, 0, sizeof(struct timeval));

usr.sbin/nscd/config.h

@@ -44,9 +44,11 @@
 
 #define DEFAULT_POSITIVE_ELEMENTS_SIZE	(2048)
 #define DEFAULT_POSITIVE_LIFETIME 	(3600)
+#define DEFAULT_POSITIVE_CONF_THRESH 	(1)
 
 #define DEFAULT_NEGATIVE_ELEMENTS_SIZE	(2048)
 #define DEFAULT_NEGATIVE_LIFETIME	(60)
+#define DEFAULT_NEGATIVE_CONF_THRESH 	(1) /* (2) ??? */
 
 #define DEFAULT_MULTIPART_ELEMENTS_SIZE	(1024 * 8)
 #define DEFAULT_MULITPART_SESSIONS_SIZE	(1024)

usr.sbin/nscd/nscd.conf.5

@@ -102,6 +102,17 @@ The same as the positive-policy, but this one is applied to the negative
 elements of the given
 .Ar cachename .
 The default policy is fifo.
+.It Va negative-confidence-threshold Oo Ar cachename Oc Op Ar value
+The number of times a query must have failed before the cache accepts
+that the element can not be found.
+At the default value of 1 each negative query result is cached and 
+immediately returned from the cache on further queries.
+Higher numbers cause queries to be retried at the configured data 
+sources the given number of times, before the negative result is 
+returned from the cache on further queries.
+This allows to probe for the existence of an entry, and then to create 
+it if it did not exist, without the negative probe result preventing 
+access to the new entry for the duration of the negative TTL.
 .It Va suggested-size Oo Ar cachename Oc Op Ar value
 This is the internal hash table size.
 The value should be a prime number for optimum performance.

usr.sbin/nscd/parser.c

@@ -167,6 +167,38 @@ set_negative_time_to_live(struct configuration *config,
 	TRACE_OUT(set_negative_time_to_live);
 }
 
+static void
+set_positive_confidence_threshold(struct configuration *config,
+	const char *entry_name, int conf_thresh)
+{
+	struct configuration_entry *entry;
+
+	TRACE_IN(set_positive_conf_thresh);
+	assert(conf_thresh > 0);
+	assert(entry_name != NULL);
+
+	entry = find_create_entry(config, entry_name);
+	assert(entry != NULL);
+	entry->positive_cache_params.confidence_threshold = conf_thresh;
+
+	TRACE_OUT(set_positive_conf_thresh);
+}
+
+static void
+set_negative_confidence_threshold(struct configuration *config,
+	const char *entry_name, int conf_thresh)
+{
+	struct configuration_entry *entry;
+
+	TRACE_IN(set_negative_conf_thresh);
+	assert(conf_thresh > 0);
+	assert(entry_name != NULL);
+	entry = find_create_entry(config, entry_name);
+	assert(entry != NULL);
+	entry->negative_cache_params.confidence_threshold = conf_thresh;
+	TRACE_OUT(set_negative_conf_thresh);
+}
+
 /*
  * Hot count is actually the elements size limit.
  */
@@ -393,6 +425,12 @@ parse_config_file(struct configuration *config,
 					fields[1], value);
 				continue;
 			} else if ((field_count == 3) &&
+			(strcmp(fields[0], "positive-confidence-threshold") == 0) &&
+			((value = get_number(fields[2], 1, -1)) != -1)) {
+				set_positive_confidence_threshold(config,
+					fields[1], value);
+				continue;
+			} else if ((field_count == 3) &&
 			(strcmp(fields[0], "positive-policy") == 0) &&
 			(check_cachename(fields[1]) == 0) &&
 			((value = get_policy(fields[2])) != -1)) {
@@ -416,6 +454,12 @@ parse_config_file(struct configuration *config,
 					fields[1], value);
 				continue;
 			} else if ((field_count == 3) &&
+			(strcmp(fields[0], "negative-confidence-threshold") == 0) &&
+			((value = get_number(fields[2], 1, -1)) != -1)) {
+				set_negative_confidence_threshold(config,
+					fields[1], value);
+				continue;
+			} else if ((field_count == 3) &&
 			(strcmp(fields[0], "negative-policy") == 0) &&
 			(check_cachename(fields[1]) == 0) &&
 			((value = get_policy(fields[2])) != -1)) {