Added endurance run to check all objects in the git source repository, something like a primtive git-fsck, which indeed takes a while to run. Its useful to see the memory consumption, which must stay static in the domain of physical memory

Author: Byron

db/pack.py

@@ -157,11 +157,15 @@ def update_pack_entity_cache(self, force=False):
 		# reinitialize prioritiess
 		self._sort_entities()
 		return True
+		
+	def entities(self):
+		""":return: list of pack entities operated upon by this database"""
+		return [ item[1] for item in self._entities ]
 
 	def sha_iter(self):
 		"""Return iterator yielding 20 byte shas for the packed objects in this data base"""
 		sha_list = list()
-		for entity in (item[1] for item in self._entities):
+		for entity in self.entities():
 			index = entity.index()
 			sha_by_index = index.sha
 			for index in xrange(index.size()):

test/performance/test_db.py test/performance/test_pack.pytest/performance/test_db.py renamed to test/performance/test_pack.py

@@ -10,7 +10,7 @@
 from time import time
 import random
 
-class TestGitDBPerformance(TestBigRepoR):
+class TestPackedDBPerformance(TestBigRepoR):
 
 	def test_pack_random_access(self):
 		pdb = PackedDB(os.path.join(self.gitrepopath, "objects/pack"))
@@ -22,7 +22,6 @@ def test_pack_random_access(self):
 		ns = len(sha_list)
 		print >> sys.stderr, "PDB: looked up %i shas by index in %f s ( %f shas/s )" % (ns, elapsed, ns / elapsed)
 
-		
 		# sha lookup: best-case and worst case access
 		pdb_pack_info = pdb._pack_info
 		access_times = list()
@@ -39,7 +38,7 @@ def test_pack_random_access(self):
 
 			# discard cache
 			del(pdb._entities)
-			pdb._entities
+			pdb.entities()
 			print >> sys.stderr, "PDB: looked up %i sha (random=%i) in %f s ( %f shas/s )" % (ns, rand, elapsed, ns / elapsed)
 		# END for each random mode
 		elapsed_order, elapsed_rand = access_times
@@ -72,3 +71,24 @@ def test_pack_random_access(self):
 		total_kib = total_size / 1000
 		print >> sys.stderr, "PDB: Obtained %i streams by sha and read all bytes totallying %i KiB ( %f KiB / s ) in %f s ( %f streams/s )" % (max_items, total_kib, total_kib/elapsed , elapsed, max_items / elapsed)
 
+	
+		print >> sys.stderr, "Endurance run: verify streaming of %i objects (crc and sha)" % ns
+		for crc in range(2):
+			count = 0
+			st = time()
+			for entity in pdb.entities():
+				pack_verify = entity.is_valid_stream
+				sha_by_index = entity.index().sha
+				for index in xrange(entity.index().size()):
+					try:
+						assert pack_verify(sha_by_index(index), use_crc=crc)
+					except UnsupportedOperation:
+						pass
+					# END ignore old indices
+					count += 1
+				# END for each index
+			# END for each entity
+			elapsed = time() - st
+			print >> sys.stderr, "PDB: verified %i objects (crc=%i) in %f s ( %f objects/s )" % (count, crc, elapsed, count / elapsed)
+		# END for each verify mode
+