diff --git a/pack-objects.c b/pack-objects.c
index 52560293b6..24ef42fc0a 100644
--- a/pack-objects.c
+++ b/pack-objects.c
@@ -123,7 +123,10 @@ void oe_map_new_pack(struct packing_data *pack)
 {
 	uint32_t i;
 
-	REALLOC_ARRAY(pack->in_pack, pack->nr_alloc);
+	if (pack->in_pack)
+		BUG("packing_data has already been converted to pack array");
+
+	ALLOC_ARRAY(pack->in_pack, pack->nr_alloc);
 
 	for (i = 0; i < pack->nr_objects; i++)
 		pack->in_pack[i] = oe_in_pack(pack, pack->objects + i);
diff --git a/pack-objects.h b/pack-objects.h
index 857d43850b..9f55ac9f91 100644
--- a/pack-objects.h
+++ b/pack-objects.h
@@ -253,12 +253,21 @@ static inline void oe_set_in_pack(struct packing_data *pack,
 				  struct object_entry *e,
 				  struct packed_git *p)
 {
-	if (!p->index)
+	if (pack->in_pack_by_idx) {
+		if (p->index) {
+			e->in_pack_idx = p->index;
+			return;
+		}
+		/*
+		 * We're accessing packs by index, but this pack doesn't have
+		 * an index (e.g., because it was added since we created the
+		 * in_pack_by_idx array). Bail to oe_map_new_pack(), which
+		 * will convert us to using the full in_pack array, and then
+		 * fall through to our in_pack handling.
+		 */
 		oe_map_new_pack(pack);
-	if (pack->in_pack_by_idx)
-		e->in_pack_idx = p->index;
-	else
-		pack->in_pack[e - pack->objects] = p;
+	}
+	pack->in_pack[e - pack->objects] = p;
 }
 
 static inline struct object_entry *oe_delta(
diff --git a/t/perf/p5303-many-packs.sh b/t/perf/p5303-many-packs.sh
index 3779851941..a369152c47 100755
--- a/t/perf/p5303-many-packs.sh
+++ b/t/perf/p5303-many-packs.sh
@@ -77,6 +77,7 @@ do
 	# actual pack generation, without smudging the on-disk setup
 	# between trials.
 	test_perf "repack ($nr_packs)" '
+		GIT_TEST_FULL_IN_PACK_ARRAY=1 \
 		git pack-objects --keep-true-parents \
 		  --honor-pack-keep --non-empty --all \
 		  --reflog --indexed-objects --delta-base-offset \