Optimize package registration/deregistration. Previously, when looking up the

package name for the origin of a dependency, all entries in /var/db/pkg were
traversed for each dependency of added/removed package.  Now, gather all the
origins first, then do the lookup in a single pass over /var/db/pkg.

This should provide a major speedup for packages with hundreds of dependencies.

Submitted by:	rdivacky (earlier version)
MFC after:	1 month

usr.sbin/pkg_install/add/perform.c

@@ -452,6 +452,8 @@ pkg_do(char *pkg)
     /* Time to record the deed? */
     if (!NoRecord && !Fake) {
 	char contents[FILENAME_MAX];
+	char **depnames = NULL, **deporigins = NULL, **depmatches;
+	int i, dep_count = 0;
 	FILE *contfile;
 
 	if (getuid() != 0)
@@ -495,8 +497,7 @@ pkg_do(char *pkg)
 	write_plist(&Plist, contfile);
 	fclose(contfile);
 	for (p = Plist.head; p ; p = p->next) {
-	    char *deporigin, **depnames;
-	    int i;
+	    char *deporigin;
 
 	    if (p->type != PLIST_PKGDEP)
 		continue;
@@ -509,31 +510,68 @@ pkg_do(char *pkg)
 		printf(".\n");
 	    }
 
-	    depnames = (deporigin != NULL) ? matchbyorigin(deporigin, NULL) :
-					     NULL;
-	    if (depnames == NULL) {
-		depnames = alloca(sizeof(*depnames) * 2);
-		depnames[0] = p->name;
-		depnames[1] = NULL;
-	    }
-		if(!IgnoreDeps){
-	    for (i = 0; depnames[i] != NULL; i++) {
-		sprintf(contents, "%s/%s/%s", LOG_DIR, depnames[i],
-			REQUIRED_BY_FNAME);
-		if (strcmp(p->name, depnames[i]) != 0)
-		    warnx("warning: package '%s' requires '%s', but '%s' "
-			  "is installed", Plist.name, p->name, depnames[i]);
-		contfile = fopen(contents, "a");
-		if (!contfile)
-		    warnx("can't open dependency file '%s'!\n"
-			  "dependency registration is incomplete", contents);
-		else {
-		    fprintf(contfile, "%s\n", Plist.name);
-		    if (fclose(contfile) == EOF)
-			warnx("cannot properly close file %s", contents);
-		}
+	    if (deporigin) {
+		/* Defer to origin lookup */
+		depnames = realloc(depnames, (dep_count + 1) * sizeof(*depnames));
+		depnames[dep_count] = p->name;
+		deporigins = realloc(deporigins, (dep_count + 2) * sizeof(*deporigins));
+		deporigins[dep_count] = deporigin;
+		deporigins[dep_count + 1] = NULL;
+		dep_count++;
+	    } else {
+	       /* No origin recorded, try to register on literal package name */
+	       sprintf(contents, "%s/%s/%s", LOG_DIR, p->name,
+		     REQUIRED_BY_FNAME);
+	       contfile = fopen(contents, "a");
+	       if (!contfile) {
+		  warnx("can't open dependency file '%s'!\n"
+			"dependency registration is incomplete", contents);
+	       } else {
+		  fprintf(contfile, "%s\n", Plist.name);
+		  if (fclose(contfile) == EOF) {
+		     warnx("cannot properly close file %s", contents);
+		  }
+	       }
 	    }
 	}
+	if (dep_count > 0) {
+	    depmatches = matchallbyorigin((const char **)deporigins, NULL);
+	    free(deporigins);
+	    if (!IgnoreDeps && depmatches) {
+		for (i = 0; i < dep_count; i++) {
+		    if (depmatches[i]) {
+			/* Origin looked up */
+			sprintf(contents, "%s/%s/%s", LOG_DIR, depmatches[i],
+			    REQUIRED_BY_FNAME);
+			if (depnames[i] && strcmp(depnames[i], depmatches[i]) != 0)
+			    warnx("warning: package '%s' requires '%s', but '%s' "
+				"is installed", Plist.name, depnames[i], depmatches[i]);
+			contfile = fopen(contents, "a");
+			if (!contfile) {
+			    warnx("can't open dependency file '%s'!\n"
+				"dependency registration is incomplete", contents);
+			} else {
+			    fprintf(contfile, "%s\n", Plist.name);
+			    if (fclose(contfile) == EOF)
+				warnx("cannot properly close file %s", contents);
+			}
+		    } else if (depnames[i]) {
+			/* No package present with this origin, try literal package name */
+			sprintf(contents, "%s/%s/%s", LOG_DIR, depnames[i],
+			    REQUIRED_BY_FNAME);
+			contfile = fopen(contents, "a");
+			if (!contfile) {
+			    warnx("can't open dependency file '%s'!\n"
+				"dependency registration is incomplete", contents);
+			} else {
+			    fprintf(contfile, "%s\n", Plist.name);
+			    if (fclose(contfile) == EOF) {
+				warnx("cannot properly close file %s", contents);
+			    }
+			}
+		    }
+		}
+	    }
 	}
 	if (Verbose)
 	    printf("Package %s registered in %s\n", Plist.name, LogDir);

usr.sbin/pkg_install/delete/perform.c

@@ -122,12 +122,12 @@ static int
 pkg_do(char *pkg)
 {
     FILE *cfile;
-    char *deporigin, **depnames, home[FILENAME_MAX];
+    char *deporigin, **deporigins = NULL, **depnames = NULL, **depmatches, home[FILENAME_MAX];
     PackingList p;
     int i, len;
     int isinstalled;
     /* support for separate pre/post install scripts */
-    int new_m = 0;
+    int new_m = 0, dep_count = 0;
     const char *pre_script = DEINSTALL_FNAME;
     const char *post_script, *pre_arg, *post_arg;
     struct reqr_by_entry *rb_entry;
@@ -275,15 +275,31 @@ pkg_do(char *pkg)
 	    printf(".\n");
 	}
 	if (!Fake) {
-	    depnames = (deporigin != NULL) ? matchbyorigin(deporigin, NULL) :
-					     NULL;
-	    if (depnames == NULL) {
-		depnames = alloca(sizeof(*depnames) * 2);
-		depnames[0] = p->name;
-		depnames[1] = NULL;
+	    if (deporigin) {
+		deporigins = realloc(deporigins, (dep_count + 2) * sizeof(*deporigins));
+		depnames = realloc(depnames, (dep_count + 1) * sizeof(*depnames));
+		deporigins[dep_count] = deporigin;
+		deporigins[dep_count + 1] = NULL;
+		depnames[dep_count] = p->name;
+		dep_count++;
+	    } else {
+		undepend(p->name, pkg);
+	    }
+	}
+    }
+
+    if (dep_count > 0) {
+	/* Undepend all the dependencies at once */
+	depmatches = matchallbyorigin((const char **)deporigins, NULL);
+	free(deporigins);
+	if (depmatches) {
+	    for (i = 0; i < dep_count; i++) {
+		if (depmatches[i]) {
+		    undepend(depmatches[i], pkg);
+		} else if (depnames[i]) {
+		    undepend(depnames[i], pkg);
+		}
 	    }
-	    for (i = 0; depnames[i] != NULL; i++)
-		undepend(depnames[i], pkg);
 	}
     }
 

usr.sbin/pkg_install/lib/lib.h

@@ -225,6 +225,7 @@ int		real_main(int, char **);
 /* Query installed packages */
 char		**matchinstalled(match_t, char **, int *);
 char		**matchbyorigin(const char *, int *);
+char		**matchallbyorigin(const char **, int *);
 int		isinstalledpkg(const char *name);
 int		pattern_match(match_t MatchType, char *pattern, const char *pkgname);
 

usr.sbin/pkg_install/lib/match.c

@@ -238,18 +238,10 @@ pattern_match(match_t MatchType, char *pattern, const char *pkgname)
  * as a key for matching packages.
  */
 char **
-matchbyorigin(const char *origin, int *retval)
+matchallbyorigin(const char **origins, int *retval)
 {
-    char **installed;
-    int i;
-    static struct store *store = NULL;
-
-    store = storecreate(store);
-    if (store == NULL) {
-	if (retval != NULL)
-	    *retval = 1;
-	return NULL;
-    }
+    char **installed, **allorigins = NULL, **matches = NULL;
+    int i, j;
 
     if (retval != NULL)
 	*retval = 0;
@@ -258,11 +250,15 @@ matchbyorigin(const char *origin, int *retval)
     if (installed == NULL)
 	return NULL;
 
+    /* Gather origins for all installed packages */
     for (i = 0; installed[i] != NULL; i++) {
 	FILE *fp;
-	char *cp, tmp[PATH_MAX];
+	char *buf, *cp, tmp[PATH_MAX];
 	int cmd;
 
+	allorigins = realloc(allorigins, (i + 1) * sizeof(*allorigins));
+	allorigins[i] = NULL;
+
 	snprintf(tmp, PATH_MAX, "%s/%s", LOG_DIR, installed[i]);
 	/*
 	 * SPECIAL CASE: ignore empty dirs, since we can can see them
@@ -290,8 +286,8 @@ matchbyorigin(const char *origin, int *retval)
 		continue;
 	    cmd = plist_cmd(tmp + 1, &cp);
 	    if (cmd == PLIST_ORIGIN) {
-		if (csh_match(origin, cp, FNM_PATHNAME) == 0)
-		    storeappend(store, installed[i]);
+		asprintf(&buf, "%s", cp);
+		allorigins[i] = buf;
 		break;
 	    }
 	}
@@ -300,10 +296,52 @@ matchbyorigin(const char *origin, int *retval)
 	fclose(fp);
     }
 
-    if (store->used == 0)
+    /* Resolve origins into package names, retaining the sequence */
+    for (i = 0; origins[i] != NULL; i++) {
+	matches = realloc(matches, (i + 1) * sizeof(*matches));
+	matches[i] = NULL;
+
+	for (j = 0; installed[j] != NULL; j++) {
+	    if (allorigins[j]) {
+		if (csh_match(origins[i], allorigins[j], FNM_PATHNAME) == 0) {
+		    matches[i] = installed[j];
+		    break;
+		}
+	    }
+	}
+    }
+
+    if (allorigins) {
+	for (i = 0; installed[i] != NULL; i++)
+	    if (allorigins[i])
+		free(allorigins[i]);
+	free(allorigins);
+    }
+
+    return matches;
+}
+
+/*
+ * Synopsis is similar to matchinstalled(), but use origin
+ * as a key for matching packages.
+ */
+char **
+matchbyorigin(const char *origin, int *retval)
+{
+   const char *origins[2];
+   char **tmp;
+
+   origins[0] = origin;
+   origins[1] = NULL;
+
+   tmp = matchallbyorigin(origins, retval);
+   if (tmp && tmp[0]) {
+	tmp = realloc(tmp, 2 * sizeof(*tmp));
+	tmp[1] = NULL;
+	return tmp;
+   } else {
 	return NULL;
-    else
-	return store->store;
+   }
 }
 
 /*