Added initial test for putting some often-called functions into an extension module, for now its only being built by a cheap hardcoded makefile. It shows that the performance gain is rather small, bottlenecks are attr accesses, so in fact the whole type wants to be put into C to get real performance. Its not really worth it for 25% I believe

Author: Byron

.gitignore

@@ -1 +1,3 @@
 *.pyc
+*.o
+*.so

_fun.c

@@ -0,0 +1,97 @@
+#include <Python.h>
+#include <stdint.h>
+
+static PyObject *PackIndexFile_sha_to_index(PyObject *self, PyObject *args)
+{
+	const unsigned char *sha;
+	const unsigned int sha_len;
+	
+	// Note: self is only set if we are a c type. We emulate an instance method, 
+	// hence we have to get the instance as 'first' argument
+	
+	// get instance and sha
+	PyObject* inst = 0;
+	if (!PyArg_ParseTuple(args, "Os#", &inst, &sha, &sha_len))
+		return NULL;
+	
+	if (sha_len != 20) {
+		PyErr_SetString(PyExc_ValueError, "Sha is not 20 bytes long");
+		return NULL;
+	}
+	
+	if( !inst){
+		PyErr_SetString(PyExc_ValueError, "Cannot be called without self");
+		return NULL;
+	}
+	
+	// read lo and hi bounds
+	PyObject* fanout_table = PyObject_GetAttrString(inst, "_fanout_table");
+	if (!fanout_table){
+		PyErr_SetString(PyExc_ValueError, "Couldn't obtain fanout table");
+		return NULL;
+	}
+	
+	unsigned int lo = 0, hi = 0;
+	if (sha[0]){
+		PyObject* item = PySequence_GetItem(fanout_table, (const Py_ssize_t)(sha[0]-1));
+		lo = PyInt_AS_LONG(item);
+		Py_DECREF(item);
+	}
+	PyObject* item = PySequence_GetItem(fanout_table, (const Py_ssize_t)sha[0]);
+	hi = PyInt_AS_LONG(item);
+	Py_DECREF(item);
+	item = 0;
+	
+	Py_DECREF(fanout_table);
+	
+	// get sha query function
+	PyObject* get_sha = PyObject_GetAttrString(inst, "sha");
+	if (!get_sha){
+		PyErr_SetString(PyExc_ValueError, "Couldn't obtain sha method");
+		return NULL;
+	}
+	
+	PyObject *sha_str = 0;
+	while (lo < hi) {
+		const int mid = (lo + hi)/2;
+		sha_str = PyObject_CallFunction(get_sha, "i", mid);
+		if (!sha_str) {
+			return NULL;
+		}
+		
+		// we really trust that string ... for speed 
+		const int cmp = memcmp(PyString_AS_STRING(sha_str), sha, 20);
+		Py_DECREF(sha_str);
+		sha_str = 0;
+		
+		if (cmp < 0){
+			lo = mid + 1;
+		}
+		else if (cmp > 0) {
+			hi = mid;
+		}
+		else {
+			Py_DECREF(get_sha);
+			return PyInt_FromLong(mid);
+		}// END handle comparison
+	}// END while lo < hi
+	
+	// nothing found, cleanup
+	Py_DECREF(get_sha);
+	Py_RETURN_NONE;
+}
+
+
+static PyMethodDef py_fun[] = {
+	{ "PackIndexFile_sha_to_index", (PyCFunction)PackIndexFile_sha_to_index, METH_VARARGS, NULL },
+	{ NULL, NULL, 0, NULL }
+};
+
+void init_fun(void)
+{
+	PyObject *m;
+
+	m = Py_InitModule3("_fun", py_fun, NULL);
+	if (m == NULL)
+		return;
+}

makefile

@@ -0,0 +1,12 @@
+
+_fun.o: _fun.c
+	gcc -pthread -fno-strict-aliasing -DNDEBUG -g -fwrapv -O2 -Wall -Wstrict-prototypes -fPIC -I/usr/include/python2.6 -c $< -o $@
+
+_fun.so: _fun.o
+	gcc -pthread -shared -Wl,-O1 -Wl,-Bsymbolic-functions $^ -o $@
+	
+all: _fun.so
+
+clean:
+	-rm *.so
+	-rm *.o

pack.py

@@ -23,6 +23,12 @@
 					msb_size
 				)
 
+try:
+	from _fun import PackIndexFile_sha_to_index
+except ImportError:
+	pass
+# END try c module
+
 from base import (		# Amazing !
 						OInfo,
 						OStream,
@@ -298,6 +304,13 @@ def sha_to_index(self, sha):
 		# END bisect
 		return None
 
+	if 'PackIndexFile_sha_to_index' in globals():
+		# NOTE: Its just about 25% faster, the major bottleneck might be the attr 
+		# accesses
+		def sha_to_index(self, sha):
+			return PackIndexFile_sha_to_index(self, sha)
+	# END redefine heavy-hitter with c version  
+	
 	#} END properties